jscpd-rs 0.1.0

package/examples/dump_oxc_tokens.rs

@@ -0,0 +1,112 @@
+use std::{env, fs, path::Path};
+
+use oxc_allocator::Allocator;
+use oxc_parser::{
+    Parser,
+    config::{TokensLexerConfig, TokensParserConfig},
+    lexer::Lexer,
+};
+use oxc_span::SourceType;
+
+fn main() {
+    let args = env::args().collect::<Vec<_>>();
+    if args.len() < 2 {
+        eprintln!("usage: dump_oxc_tokens [--lexer] <file> [start-line] [end-line]");
+        std::process::exit(2);
+    }
+
+    let use_lexer = args.get(1).is_some_and(|arg| arg == "--lexer");
+    let first_positional = if use_lexer { 2 } else { 1 };
+    let path = &args[first_positional];
+    let start_line = args
+        .get(first_positional + 1)
+        .and_then(|value| value.parse().ok())
+        .unwrap_or(1);
+    let end_line = args
+        .get(first_positional + 2)
+        .and_then(|value| value.parse().ok())
+        .unwrap_or(usize::MAX);
+    let content = fs::read_to_string(path).expect("read file");
+    let source_type = SourceType::from_path(Path::new(path)).unwrap_or_default();
+    let allocator = Allocator::new();
+    let line_index = LineIndex::new(&content);
+    let mut count = 0usize;
+
+    if use_lexer {
+        let mut lexer =
+            Lexer::new_for_benchmarks(&allocator, &content, source_type, TokensLexerConfig);
+        let mut token = lexer.first_token();
+        while !token.kind().is_eof() {
+            let start = token.start() as usize;
+            let end = token.end() as usize;
+            let location = line_index.location(start);
+            if location.line >= start_line && location.line <= end_line {
+                println!(
+                    "{}:{} {:?}:{}",
+                    location.line,
+                    location.column,
+                    token.kind(),
+                    &content[start..end]
+                );
+            }
+            count += 1;
+            token = lexer.next_token_for_benchmarks();
+        }
+    } else {
+        let parser_return = Parser::new(&allocator, &content, source_type)
+            .with_config(TokensParserConfig)
+            .parse();
+        for token in parser_return.tokens {
+            let start = token.start() as usize;
+            let end = token.end() as usize;
+            let location = line_index.location(start);
+            if location.line >= start_line && location.line <= end_line {
+                println!(
+                    "{}:{} {:?}:{}",
+                    location.line,
+                    location.column,
+                    token.kind(),
+                    &content[start..end]
+                );
+            }
+            count += 1;
+        }
+    }
+    eprintln!("tokens: {count}");
+}
+
+struct LineIndex {
+    newlines: Vec<usize>,
+}
+
+impl LineIndex {
+    fn new(content: &str) -> Self {
+        Self {
+            newlines: content
+                .bytes()
+                .enumerate()
+                .filter_map(|(idx, byte)| (byte == b'\n').then_some(idx))
+                .collect(),
+        }
+    }
+
+    fn location(&self, offset: usize) -> Location {
+        let previous_newlines = self
+            .newlines
+            .partition_point(|newline_offset| *newline_offset < offset);
+        let line_start = if previous_newlines == 0 {
+            0
+        } else {
+            self.newlines[previous_newlines - 1] + 1
+        };
+        Location {
+            line: previous_newlines + 1,
+            column: offset - line_start + 1,
+        }
+    }
+}
+
+struct Location {
+    line: usize,
+    column: usize,
+}

package/examples/library_api.rs

@@ -0,0 +1,42 @@
+use jscpd_rs::{SourceFile, detect_source_files, get_default_options};
+
+fn main() {
+    let mut options = get_default_options();
+    options.reporters.clear();
+    options.silent = true;
+    options.no_tips = true;
+    options.min_lines = 2;
+    options.min_tokens = 5;
+
+    let files = vec![
+        SourceFile {
+            source_id: "a.js".to_string(),
+            format: "javascript".to_string(),
+            content: duplicate_body(),
+        },
+        SourceFile {
+            source_id: "b.js".to_string(),
+            format: "javascript".to_string(),
+            content: duplicate_body(),
+        },
+    ];
+
+    let result = detect_source_files(files, &options);
+    println!(
+        "{} clones, {} duplicated lines, {:.2}% duplicated",
+        result.clones.len(),
+        result.statistics.total.duplicated_lines,
+        result.statistics.total.percentage
+    );
+}
+
+fn duplicate_body() -> String {
+    [
+        "const alpha = 1;",
+        "const beta = 2;",
+        "const gamma = alpha + beta;",
+        "console.log(gamma);",
+        "",
+    ]
+    .join("\n")
+}

package/npm/bin/jscpd-rs.js

@@ -0,0 +1,6 @@
+#!/usr/bin/env node
+"use strict";
+
+const { runBinary } = require("../lib/run-binary");
+
+runBinary("jscpd", process.argv.slice(2));

package/npm/bin/jscpd-server.js

@@ -0,0 +1,6 @@
+#!/usr/bin/env node
+"use strict";
+
+const { runBinary } = require("../lib/run-binary");
+
+runBinary("jscpd-server", process.argv.slice(2));

package/npm/lib/run-binary.js

@@ -0,0 +1,68 @@
+"use strict";
+
+const fs = require("node:fs");
+const path = require("node:path");
+const { spawnSync } = require("node:child_process");
+
+function packageRoot() {
+  return path.resolve(__dirname, "..", "..");
+}
+
+function binaryPath(name) {
+  const exe = process.platform === "win32" ? `${name}.exe` : name;
+  const targetDir =
+    process.env.CARGO_TARGET_DIR || path.join(packageRoot(), "target");
+  return path.join(targetDir, "release", exe);
+}
+
+function buildIfMissing(name) {
+  const binary = binaryPath(name);
+  if (fs.existsSync(binary)) {
+    return binary;
+  }
+
+  const script = path.join(packageRoot(), "npm", "scripts", "postinstall.js");
+  const result = spawnSync(process.execPath, [script], {
+    cwd: packageRoot(),
+    stdio: "inherit",
+    env: process.env,
+  });
+
+  if (result.error) {
+    throw result.error;
+  }
+  if (result.signal) {
+    process.kill(process.pid, result.signal);
+  }
+  if (result.status !== 0) {
+    process.exit(result.status ?? 1);
+  }
+  if (!fs.existsSync(binary)) {
+    console.error(`jscpd-rs: expected binary was not built: ${binary}`);
+    process.exit(1);
+  }
+
+  return binary;
+}
+
+function runBinary(name, args) {
+  const binary = buildIfMissing(name);
+  const result = spawnSync(binary, args, {
+    stdio: "inherit",
+    env: process.env,
+  });
+
+  if (result.error) {
+    if (result.error.code === "ENOENT") {
+      console.error(`jscpd-rs: binary not found: ${binary}`);
+      process.exit(1);
+    }
+    throw result.error;
+  }
+  if (result.signal) {
+    process.kill(process.pid, result.signal);
+  }
+  process.exit(result.status ?? 0);
+}
+
+module.exports = { runBinary };

package/npm/scripts/postinstall.js

@@ -0,0 +1,50 @@
+"use strict";
+
+const fs = require("node:fs");
+const path = require("node:path");
+const { spawnSync } = require("node:child_process");
+
+const root = path.resolve(__dirname, "..", "..");
+const cargo = process.env.CARGO || "cargo";
+const targetDir = process.env.CARGO_TARGET_DIR || path.join(root, "target");
+const releaseDir = path.join(targetDir, "release");
+const exeSuffix = process.platform === "win32" ? ".exe" : "";
+const binaries = ["jscpd", "jscpd-server"].map((name) =>
+  path.join(releaseDir, `${name}${exeSuffix}`),
+);
+
+if (process.env.JSCPD_RS_SKIP_POSTINSTALL === "1") {
+  console.log("jscpd-rs: skipping native build because JSCPD_RS_SKIP_POSTINSTALL=1");
+  process.exit(0);
+}
+
+if (binaries.every((binary) => fs.existsSync(binary))) {
+  process.exit(0);
+}
+
+console.log("jscpd-rs: building native binaries with Cargo");
+const result = spawnSync(
+  cargo,
+  ["build", "--release", "--locked", "--bin", "jscpd", "--bin", "jscpd-server"],
+  {
+    cwd: root,
+    stdio: "inherit",
+    env: process.env,
+  },
+);
+
+if (result.error) {
+  if (result.error.code === "ENOENT") {
+    console.error(
+      "jscpd-rs: Cargo was not found. Install Rust from https://rustup.rs/ and retry.",
+    );
+    process.exit(1);
+  }
+  throw result.error;
+}
+
+if (result.signal) {
+  process.kill(process.pid, result.signal);
+}
+
+process.exit(result.status ?? 0);

package/package.json

@@ -0,0 +1,53 @@
+{
+  "name": "jscpd-rs",
+  "version": "0.1.0",
+  "description": "Fast native Rust clone of jscpd for duplicate-code detection",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/vv-bogdanov/jscpd-rs.git"
+  },
+  "homepage": "https://github.com/vv-bogdanov/jscpd-rs#readme",
+  "bugs": {
+    "url": "https://github.com/vv-bogdanov/jscpd-rs/issues"
+  },
+  "keywords": [
+    "jscpd",
+    "copy-paste",
+    "copy-paste-detector",
+    "duplication",
+    "duplicate-code",
+    "code-duplication",
+    "clone-detection",
+    "ci",
+    "sarif",
+    "cli"
+  ],
+  "bin": {
+    "jscpd-rs": "npm/bin/jscpd-rs.js",
+    "jscpd": "npm/bin/jscpd-rs.js",
+    "jscpd-server": "npm/bin/jscpd-server.js"
+  },
+  "scripts": {
+    "postinstall": "node npm/scripts/postinstall.js",
+    "test:npm-package": "scripts/npm-package-check.sh"
+  },
+  "files": [
+    "CHANGELOG.md",
+    "Cargo.lock",
+    "Cargo.toml",
+    "docs/**",
+    "examples/**",
+    "LICENSE",
+    "README.md",
+    "npm/**",
+    "skills/**",
+    "src/**"
+  ],
+  "engines": {
+    "node": ">=18"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}

package/skills/dry-refactoring/SKILL.md

@@ -0,0 +1,63 @@
+---
+name: dry-refactoring
+description: Guided workflow to eliminate copy-paste duplication detected by jscpd-rs.
+---
+
+# dry-refactoring
+
+Guided workflow to eliminate copy-paste duplication in source code. Use after
+running `jscpd-rs` to detect clones.
+
+## Prerequisites
+
+First, run `jscpd-rs` to identify duplications:
+
+```bash
+npx jscpd-rs --reporters ai <path>
+```
+
+See the [jscpd-rs skill](../jscpd/SKILL.md) for option details.
+
+## Workflow
+
+1. Run `jscpd-rs` with `--reporters ai` on the target path.
+2. Parse each clone line to identify the duplicated file and line ranges.
+3. Read both code fragments from the source files.
+4. Understand what the duplicated code does.
+5. Design a refactoring: extract a shared function, class, module, constant, or
+   base abstraction.
+6. Apply the refactoring and update all call sites, not just the two reported
+   locations.
+7. Run tests for the touched area.
+8. Re-run `jscpd-rs` to confirm the clone is gone or reduced.
+9. Repeat for the highest-impact remaining clones.
+
+## Refactoring Strategies
+
+Extract function: use when the duplicate is a repeated logic block.
+
+Extract module or utility: use when the duplicate spans files that can depend on
+a shared helper.
+
+Extract constant or config: use when the duplicate is repeated data,
+configuration, selectors, strings, or magic values.
+
+Template or base abstraction: use when the duplicate is structural and the
+shared shape is stable.
+
+## Guardrails
+
+- Do not refactor unrelated behavior while removing duplication.
+- Keep the extracted abstraction named after the domain concept, not the clone.
+- Check all similar call sites, because one clone pair can represent a larger
+  duplicated family.
+- Keep test fixtures duplicated when the duplication is intentional and improves
+  readability.
+- Prefer small, reversible refactors over broad rewrites.
+
+## Useful Commands
+
+```bash
+npx jscpd-rs --reporters ai --min-lines 10 <path>
+npx jscpd-rs --reporters json --output /tmp/jscpd-report <path>
+```

package/skills/jscpd/SKILL.md

@@ -0,0 +1,85 @@
+---
+name: jscpd
+description: Fast native Rust clone of jscpd. Detect duplicated code and measure duplication percentages.
+---
+
+# jscpd-rs
+
+Fast native Rust clone of jscpd. Use this skill to run `jscpd-rs` and
+understand its output.
+
+## Quick Start
+
+```bash
+# Run with ai reporter (compact output optimized for agents)
+npx jscpd-rs --reporters ai <path>
+
+# With ignore patterns
+npx jscpd-rs --reporters ai --ignore "**/node_modules/**,**/dist/**" <path>
+
+# Scope to specific formats
+npx jscpd-rs --reporters ai --format "javascript,typescript" <path>
+```
+
+## AI Reporter Output Format
+
+The `ai` reporter produces compact, token-efficient output designed for agent
+consumption:
+
+```text
+Clones:
+src/ foo.ts:10-25 ~ bar.ts:42-57
+src/utils/helpers.ts:100-120 ~ src/utils/other.ts:5-25
+---
+3 clones · 4.2% duplication
+```
+
+Each line represents one clone pair:
+
+- Same file: `path/file.ts 10-25 ~ 45-60`
+- Same directory: `shared/prefix/ file-a.ts:10-25 ~ file-b.ts:42-57`
+- Different paths: `path/a.ts:10-25 ~ path/b.ts:42-57`
+
+## Options
+
+| Option | Description |
+| --- | --- |
+| `--reporters ai` | Use the AI-optimized reporter |
+| `--reporters html` | Generate HTML report |
+| `--reporters json` | Output JSON report |
+| `--min-tokens N` | Minimum tokens to consider a duplication |
+| `--min-lines N` | Minimum lines to consider a duplication |
+| `--threshold N` | Exit with error if duplication percentage exceeds N |
+| `--ignore "glob"` | Ignore patterns, comma-separated |
+| `--format "list"` | Limit to specific languages |
+| `--pattern "glob"` | Glob pattern to select files |
+| `--gitignore` | Respect `.gitignore` |
+| `--output "path"` | Directory to write reports to |
+| `--silent` | Suppress output |
+| `--no-tips` | Disable terminal tips |
+| `--config "path"` | Path to `.jscpd.json` config file |
+
+## Configuration File
+
+Create a `.jscpd.json` in your project root:
+
+```json
+{
+  "threshold": 0,
+  "reporters": ["ai"],
+  "ignore": ["**/node_modules/**", "**/dist/**", "**/*.min.*"],
+  "format": ["typescript", "javascript"],
+  "minLines": 5,
+  "minTokens": 50,
+  "output": "./reports/jscpd"
+}
+```
+
+## Refactoring Duplicated Code
+
+Once duplicates are detected, use the `dry-refactoring` skill for a guided
+workflow to eliminate them:
+
+```bash
+npx skills add vv-bogdanov/jscpd-rs --skill dry-refactoring
+```