jscpd-rs 0.1.0

package/src/detector/tests.rs

@@ -0,0 +1,238 @@
+use std::collections::BTreeSet;
+
+use crate::cli::Options;
+use crate::files::SourceFile;
+use crate::tokenizer::Location;
+
+use super::{
+    CloneMatch, Fragment, dedup_exact_clones, detect, detect_prepared_drafts, prepare_source_drafts,
+};
+
+fn detection_options() -> Options {
+    Options {
+        min_tokens: 3,
+        min_lines: 0,
+        ..Options::default()
+    }
+}
+
+#[test]
+fn detects_cross_file_duplicates() {
+    let content = "alpha beta gamma delta epsilon\n";
+    let files = vec![
+        source("a.js", content),
+        source("b.js", &format!("prefix\n{content}\nsuffix\n")),
+    ];
+
+    let result = detect(files, &detection_options());
+
+    assert!(!result.clones.is_empty());
+}
+
+#[test]
+fn detects_generic_format_duplicates() {
+    let content = "alpha beta gamma delta epsilon\n";
+    let files = vec![
+        source_with_format("a.css", "css", content),
+        source_with_format("b.css", "css", &format!("prefix\n{content}\nsuffix\n")),
+    ];
+
+    let result = detect(files, &detection_options());
+
+    assert!(!result.clones.is_empty());
+}
+
+#[test]
+fn skip_local_skips_clones_inside_same_configured_root() {
+    let options = Options {
+        paths: vec!["project".into()],
+        skip_local: true,
+        min_tokens: 3,
+        min_lines: 0,
+        ..Options::default()
+    };
+    let content = "alpha beta gamma delta epsilon\n";
+    let files = vec![
+        source("project/dir1/a.js", content),
+        source("project/dir2/b.js", content),
+    ];
+
+    let result = detect(files, &options);
+
+    assert!(result.clones.is_empty());
+}
+
+#[test]
+fn skip_local_keeps_clones_across_configured_roots() {
+    let options = Options {
+        paths: vec!["left".into(), "right".into()],
+        skip_local: true,
+        min_tokens: 3,
+        min_lines: 0,
+        ..Options::default()
+    };
+    let content = "alpha beta gamma delta epsilon\n";
+    let files = vec![source("left/a.js", content), source("right/b.js", content)];
+
+    let result = detect(files, &options);
+
+    assert!(!result.clones.is_empty());
+}
+
+#[test]
+fn skips_empty_token_sources_in_statistics() {
+    let content = "// jscpd:ignore-start\nignored\n// jscpd:ignore-end\n";
+
+    let result = detect(vec![source("ignored.js", content)], &Options::default());
+
+    assert_eq!(result.sources.len(), 0);
+    assert_eq!(result.statistics.total.sources, 0);
+}
+
+#[test]
+fn prepared_drafts_detection_matches_direct_detection() {
+    let options = Options {
+        reporters: vec!["json".to_string()],
+        ..detection_options()
+    };
+    let content = "alpha beta gamma delta epsilon\n";
+    let files = vec![
+        source("a.js", content),
+        source("b.js", &format!("prefix\n{content}\nsuffix\n")),
+    ];
+
+    let direct = detect(files.clone(), &options);
+    let prepared = detect_prepared_drafts(prepare_source_drafts(files, &options), &options);
+
+    assert_eq!(prepared.clones.len(), direct.clones.len());
+    assert_eq!(
+        prepared.statistics.total.sources,
+        direct.statistics.total.sources
+    );
+    assert_eq!(
+        prepared.statistics.total.clones,
+        direct.statistics.total.clones
+    );
+    assert_eq!(
+        prepared.source_contents.keys().collect::<BTreeSet<_>>(),
+        direct.source_contents.keys().collect::<BTreeSet<_>>()
+    );
+}
+
+#[test]
+fn detects_typescript_template_tail_clone_before_member_name_difference() {
+    let options = Options {
+        min_tokens: 50,
+        min_lines: 5,
+        ..Options::default()
+    };
+    let content = r#"
+function first(workUnitAsyncStorage, reportResult) {
+  console.log = function (...args: Array<any>) {
+    const store = workUnitAsyncStorage.getStore()
+    reportResult({
+      type: 'console-call',
+      method: 'log',
+      input: `${store ? '[Store]' : '[No Store]'}: ${args.join(' ')}`,
+    })
+  }
+
+  require('next/dist/server/node-environment-extensions/console-exit')
+
+  workUnitAsyncStorage.run({ type: 'request' } as WorkUnitStore, () => {
+    console.log('inside')
+  })
+}
+
+function second(workUnitAsyncStorage, reportResult) {
+  console.error = function (...args: Array<any>) {
+    const store = workUnitAsyncStorage.getStore()
+    reportResult({
+      type: 'console-call',
+      method: 'error',
+      input: `${store ? '[Store]' : '[No Store]'}: ${args.join(' ')}`,
+    })
+  }
+
+  require('next/dist/server/node-environment-extensions/console-exit')
+
+  workUnitAsyncStorage.run({ type: 'request' } as WorkUnitStore, () => {
+    console.error('inside')
+  })
+}
+"#;
+
+    let result = detect(
+        vec![source_with_format("console.ts", "typescript", content)],
+        &options,
+    );
+
+    assert!(result.clones.iter().any(|clone| {
+        clone.duplication_a.start.line <= 24
+            && clone.duplication_a.end.line >= 32
+            && clone.duplication_b.start.line <= 7
+            && clone.duplication_b.end.line >= 15
+    }));
+}
+
+#[test]
+fn deduplicates_exact_clone_records() {
+    let mut clones = vec![
+        clone_with_lines("javascript", "a.js", 1, 5, "b.js", 1, 5),
+        clone_with_lines("javascript", "a.js", 1, 5, "b.js", 1, 5),
+        clone_with_lines("javascript", "a.js", 6, 10, "b.js", 6, 10),
+    ];
+
+    dedup_exact_clones(&mut clones);
+
+    assert_eq!(clones.len(), 2);
+    assert_eq!(clones[0].duplication_a.start.line, 1);
+    assert_eq!(clones[1].duplication_a.start.line, 6);
+}
+
+fn source(path: &str, content: &str) -> SourceFile {
+    source_with_format(path, "javascript", content)
+}
+
+fn source_with_format(path: &str, format: &str, content: &str) -> SourceFile {
+    SourceFile {
+        source_id: path.to_string(),
+        format: format.to_string(),
+        content: content.to_string(),
+    }
+}
+
+fn clone_with_lines(
+    format: &str,
+    source_a: &str,
+    start_a: usize,
+    end_a: usize,
+    source_b: &str,
+    start_b: usize,
+    end_b: usize,
+) -> CloneMatch {
+    CloneMatch {
+        format: format.to_string(),
+        duplication_a: fragment(source_a, start_a, end_a),
+        duplication_b: fragment(source_b, start_b, end_b),
+        tokens: 20,
+    }
+}
+
+fn fragment(source_id: &str, start: usize, end: usize) -> Fragment {
+    Fragment {
+        source_id: source_id.to_string(),
+        start: location(start, 1, start),
+        end: location(end, 1, end),
+        range: [start, end],
+        blame: None,
+    }
+}
+
+fn location(line: usize, column: usize, position: usize) -> Location {
+    Location {
+        line,
+        column,
+        position,
+    }
+}

package/src/detector.rs

@@ -0,0 +1,265 @@
+use std::collections::HashMap;
+
+use rayon::prelude::*;
+use rustc_hash::FxHashSet;
+
+use crate::cli::Options;
+use crate::files::SourceFile;
+
+mod matching;
+mod model;
+mod prepare;
+mod skip_local;
+mod statistics;
+mod store;
+#[cfg(test)]
+mod tests;
+
+#[cfg(test)]
+pub use model::FormatStatistic;
+pub(crate) use model::PreparedSourceDraft;
+pub use model::{
+    BlamedLine, BlamedLines, CloneMatch, DetectionResult, Fragment, SkippedClone, SourceSummary,
+    StatisticRow, Statistics,
+};
+pub use statistics::{Statistic, clone_lines};
+pub use store::{MemoryStore, MemoryStoreError};
+
+use matching::detect_format;
+use model::{FormatId, PreparedSource, SourceId, TokenStream};
+use prepare::{assign_formats, prepare_file_maps};
+use statistics::{finalize_percentages, update_clone_statistics, update_source_statistics};
+
+/// Incremental detector facade for native integrations.
+///
+/// For one-shot detection, prefer `detect_source_files` or
+/// `detect_clones_and_statistics`. Use this type when an integration wants to
+/// keep options and previously submitted in-memory sources together.
+#[derive(Clone, Debug)]
+pub struct Detector {
+    options: Options,
+    sources: Vec<SourceFile>,
+}
+
+impl Detector {
+    /// Create an empty detector with the provided options.
+    pub fn new(options: Options) -> Self {
+        Self {
+            options,
+            sources: Vec::new(),
+        }
+    }
+
+    /// Create a detector preloaded with in-memory sources.
+    pub fn with_sources(options: Options, sources: Vec<SourceFile>) -> Self {
+        Self { options, sources }
+    }
+
+    /// Return detector options.
+    pub fn options(&self) -> &Options {
+        &self.options
+    }
+
+    /// Mutably access detector options.
+    pub fn options_mut(&mut self) -> &mut Options {
+        &mut self.options
+    }
+
+    /// Return sources currently held by this detector.
+    pub fn sources(&self) -> &[SourceFile] {
+        &self.sources
+    }
+
+    /// Remove all remembered sources.
+    pub fn clear(&mut self) {
+        self.sources.clear();
+    }
+
+    /// Add one source and return clones involving that new source.
+    pub fn detect(
+        &mut self,
+        source_id: impl Into<String>,
+        text: impl Into<String>,
+        format: impl Into<String>,
+    ) -> Vec<CloneMatch> {
+        self.detect_source_file(SourceFile {
+            source_id: source_id.into(),
+            format: format.into(),
+            content: text.into(),
+        })
+    }
+
+    /// Add one prepared source and return clones involving that new source.
+    pub fn detect_source_file(&mut self, source: SourceFile) -> Vec<CloneMatch> {
+        let source_id = source.source_id.clone();
+        self.sources.push(source);
+        let result = detect(self.sources.clone(), &self.options);
+        result
+            .clones
+            .into_iter()
+            .filter(|clone| {
+                clone.duplication_a.source_id == source_id
+                    || clone.duplication_b.source_id == source_id
+            })
+            .collect()
+    }
+
+    /// Run one-shot detection against the provided prepared sources.
+    pub fn detect_files(&self, files: Vec<SourceFile>) -> DetectionResult {
+        detect(files, &self.options)
+    }
+}
+
+pub fn detect(files: Vec<SourceFile>, options: &Options) -> DetectionResult {
+    detect_prepared_drafts(prepare_source_drafts(files, options), options)
+}
+
+pub(crate) fn prepare_source_drafts(
+    files: Vec<SourceFile>,
+    options: &Options,
+) -> Vec<PreparedSourceDraft> {
+    files
+        .into_par_iter()
+        .map(|file| prepare_file_maps(file, options))
+        .collect::<Vec<_>>()
+        .into_iter()
+        .flatten()
+        .collect::<Vec<_>>()
+}
+
+pub(crate) fn detect_prepared_drafts(
+    prepared_drafts: Vec<PreparedSourceDraft>,
+    options: &Options,
+) -> DetectionResult {
+    let (format_ids, format_names) = assign_formats(&prepared_drafts);
+    let prepared_files = prepared_drafts
+        .into_iter()
+        .enumerate()
+        .map(|(idx, draft)| PreparedSource {
+            meta: draft.meta,
+            stream: TokenStream {
+                source_id: SourceId(idx),
+                format_id: format_ids[idx],
+                hashes: draft.hashes,
+                spans: draft.spans,
+            },
+        })
+        .collect::<Vec<_>>();
+
+    let mut statistics = Statistics::default();
+    let mut sources = Vec::new();
+    let mut source_contents = HashMap::new();
+    let mut source_indices_by_format = vec![Vec::new(); format_names.len()];
+    let include_source_contents = options
+        .reporters
+        .iter()
+        .any(|reporter| matches!(reporter.as_str(), "json" | "xml" | "html" | "consoleFull"));
+
+    for (idx, prepared) in prepared_files.iter().enumerate() {
+        if prepared.stream.spans.is_empty() {
+            continue;
+        }
+        update_source_statistics(
+            &mut statistics,
+            &prepared.meta.source_id,
+            &prepared.meta.format,
+            prepared.meta.lines,
+            prepared.meta.tokens,
+        );
+        sources.push(SourceSummary {
+            path: prepared.meta.source_id.clone(),
+            format: prepared.meta.format.clone(),
+            lines: prepared.meta.lines,
+            tokens: prepared.meta.tokens,
+        });
+        if include_source_contents {
+            source_contents.insert(
+                prepared.meta.source_id.clone(),
+                prepared.meta.content.clone(),
+            );
+        }
+        source_indices_by_format[prepared.stream.format_id.0].push(idx);
+    }
+
+    let format_results = source_indices_by_format
+        .par_iter()
+        .enumerate()
+        .map(|(format_id, source_indices)| {
+            detect_format(
+                FormatId(format_id),
+                source_indices,
+                &prepared_files,
+                &format_names,
+                options,
+            )
+        })
+        .collect::<Vec<_>>();
+
+    let mut clones = Vec::new();
+    let mut skipped_clones = Vec::new();
+    for format_result in format_results {
+        clones.extend(format_result.clones);
+        skipped_clones.extend(format_result.skipped_clones);
+    }
+    dedup_exact_clones(&mut clones);
+    for clone in &clones {
+        update_clone_statistics(&mut statistics, clone);
+    }
+
+    finalize_percentages(&mut statistics);
+
+    DetectionResult {
+        clones,
+        skipped_clones,
+        statistics,
+        sources,
+        source_contents,
+    }
+}
+
+fn dedup_exact_clones(clones: &mut Vec<CloneMatch>) {
+    let mut seen = FxHashSet::default();
+    clones.retain(|clone| seen.insert(CloneDedupKey::from(clone)));
+}
+
+#[derive(Hash, Eq, PartialEq)]
+struct CloneDedupKey {
+    format: String,
+    duplication_a: FragmentDedupKey,
+    duplication_b: FragmentDedupKey,
+    tokens: usize,
+}
+
+impl From<&CloneMatch> for CloneDedupKey {
+    fn from(clone: &CloneMatch) -> Self {
+        Self {
+            format: clone.format.clone(),
+            duplication_a: FragmentDedupKey::from(&clone.duplication_a),
+            duplication_b: FragmentDedupKey::from(&clone.duplication_b),
+            tokens: clone.tokens,
+        }
+    }
+}
+
+#[derive(Hash, Eq, PartialEq)]
+struct FragmentDedupKey {
+    source_id: String,
+    start_line: usize,
+    start_column: usize,
+    end_line: usize,
+    end_column: usize,
+    range: [usize; 2],
+}
+
+impl From<&Fragment> for FragmentDedupKey {
+    fn from(fragment: &Fragment) -> Self {
+        Self {
+            source_id: fragment.source_id.clone(),
+            start_line: fragment.start.line,
+            start_column: fragment.start.column,
+            end_line: fragment.end.line,
+            end_column: fragment.end.column,
+            range: fragment.range,
+        }
+    }
+}