jscpd-rs 0.1.2 → 0.1.3

package/src/cli/config.rs

@@ -501,6 +501,206 @@ mod tests {
         );
     }
 
+    #[test]
+    fn config_number_strings_follow_node_number_rules() {
+        assert_eq!(parse_config_usize_string("0x10").unwrap(), 16);
+        assert_eq!(parse_config_usize_string(" 42 ").unwrap(), 42);
+        assert!(parse_config_usize_string("1.5").is_err());
+        assert!(parse_config_usize_string("-1").is_err());
+        assert!(parse_config_usize_string("Infinity").is_err());
+
+        let config: FileConfig =
+            serde_json::from_str(r#"{"minLines":"0b101","threshold":"1e2"}"#).unwrap();
+        assert_eq!(config.min_lines, Some(5));
+        assert_eq!(config.threshold, Some(100.0));
+
+        let error = serde_json::from_str::<FileConfig>(r#"{"minLines":[]}"#).unwrap_err();
+        assert!(error.to_string().contains("invalid type: array"));
+    }
+
+    #[test]
+    fn config_number_deserializers_accept_numbers_nulls_and_reject_wrong_types() {
+        let config: FileConfig =
+            serde_json::from_str(r#"{"minLines":7,"maxLines":null,"threshold":2.5}"#).unwrap();
+        assert_eq!(config.min_lines, Some(7));
+        assert_eq!(config.max_lines, None);
+        assert_eq!(config.threshold, Some(2.5));
+
+        let config: FileConfig =
+            serde_json::from_str(r#"{"minLines":null,"threshold":null}"#).unwrap();
+        assert_eq!(config.min_lines, None);
+        assert_eq!(config.threshold, None);
+
+        let error = serde_json::from_str::<FileConfig>(r#"{"threshold":{}}"#).unwrap_err();
+        assert!(error.to_string().contains("invalid type: object"));
+    }
+
+    #[test]
+    fn clean_lexical_path_collapses_dot_and_parent_components() {
+        assert_eq!(
+            clean_lexical_path(Path::new("repo/app/./src/../tests/file.js")),
+            PathBuf::from("repo/app/tests/file.js")
+        );
+        assert_eq!(
+            clean_lexical_path(Path::new("../repo/./src")),
+            PathBuf::from("../repo/src")
+        );
+        assert!(
+            absolute_config_path(Path::new("Cargo.toml"))
+                .unwrap()
+                .is_absolute()
+        );
+    }
+
+    #[test]
+    fn resolves_config_paths_and_ignores_like_upstream() {
+        let cwd = std::env::current_dir().unwrap();
+        let config_dir = cwd.join("fixtures").join("config");
+        let absolute = cwd.join("src");
+
+        assert_eq!(
+            resolve_config_path(&config_dir, "src"),
+            config_dir.join("src")
+        );
+        assert_eq!(resolve_config_path(&config_dir, absolute.clone()), absolute);
+        assert_eq!(
+            resolve_config_ignore(&config_dir, "**/vendor/**".to_string()).unwrap(),
+            "**/vendor/**"
+        );
+        assert_eq!(
+            resolve_config_ignore(&cwd, "target/**".to_string()).unwrap(),
+            "target/**"
+        );
+
+        let absolute = cwd.join("dist/**").display().to_string();
+        assert_eq!(
+            resolve_config_ignore(&config_dir, absolute.clone()).unwrap(),
+            absolute
+        );
+    }
+
+    #[test]
+    fn applies_config_mappings_and_exit_code_variants() {
+        let config: FileConfig = serde_json::from_str(
+            r#"{
+                "format": "javascript,typescript",
+                "formatsExts": {"javascript": ["mjs", "cjs"]},
+                "formatsNames": "makefile:Makefile,GNUmakefile",
+                "exitCode": false,
+                "tokensToSkip": "import,require"
+            }"#,
+        )
+        .unwrap();
+        let mut options = Options::default();
+
+        apply_config(&mut options, config, Path::new(".")).unwrap();
+
+        assert_eq!(
+            options.format_order,
+            Some(vec!["javascript".to_string(), "typescript".to_string()])
+        );
+        assert_eq!(
+            options.formats_exts.find_format_for_value("mjs"),
+            Some("javascript")
+        );
+        assert_eq!(
+            options.formats_names.find_format_for_value("GNUmakefile"),
+            Some("makefile")
+        );
+        assert_eq!(options.exit_code, ExitCode::Boolean(false));
+        assert_eq!(options.tokens_to_skip, vec!["import", "require"]);
+    }
+
+    #[test]
+    fn node_like_json_syntax_message_handles_eof() {
+        let data = "{\"minLines\":";
+        let error = serde_json::from_str::<FileConfig>(data).unwrap_err();
+
+        assert_eq!(
+            node_like_json_syntax_message(data, &error),
+            "Unexpected end of JSON input"
+        );
+        assert_eq!(json_error_position("a\nbc", 2, 2), 3);
+
+        let trailing = "[1,]";
+        let error = serde_json::from_str::<serde_json::Value>(trailing).unwrap_err();
+        assert!(node_like_json_syntax_message(trailing, &error).contains("at position"));
+    }
+
+    #[test]
+    fn read_config_reports_syntax_errors_with_upstream_shape() {
+        let root = unique_temp_dir("jscpd-rs-config-syntax");
+        std::fs::create_dir_all(&root).unwrap();
+        let path = root.join(".jscpd.json");
+        std::fs::write(&path, "{ invalid json\n").unwrap();
+
+        let error = read_config(Some(&path)).unwrap_err();
+        let _ = std::fs::remove_dir_all(root);
+
+        assert!(error.to_string().starts_with("SyntaxError: "));
+        assert!(error.to_string().contains("Expected property name or '}'"));
+    }
+
+    #[test]
+    fn apply_config_updates_main_runtime_flags() {
+        let config: FileConfig = serde_json::from_str(
+            r#"{
+                "executionId": "run-1",
+                "path": "src,tests",
+                "pattern": "**/*.rs",
+                "ignore": "target/**,dist/**",
+                "reporters": "json,console",
+                "ignorePattern": ["import .*"],
+                "minLines": 3,
+                "minTokens": 7,
+                "maxLines": 99,
+                "maxSize": "2kb",
+                "threshold": 10,
+                "mode": "weak",
+                "silent": true,
+                "absolute": true,
+                "noSymlinks": true,
+                "ignoreCase": true,
+                "gitignore": false,
+                "debug": true,
+                "verbose": true,
+                "skipLocal": true,
+                "exitCode": 2,
+                "noTips": true
+            }"#,
+        )
+        .unwrap();
+        let mut options = Options::default();
+
+        apply_config(&mut options, config, Path::new(".")).unwrap();
+
+        assert_eq!(options.execution_id.as_deref(), Some("run-1"));
+        assert_eq!(
+            options.paths,
+            vec![PathBuf::from("./src"), PathBuf::from("./tests")]
+        );
+        assert_eq!(options.pattern, "**/*.rs");
+        assert_eq!(options.ignore, vec!["./target/**", "./dist/**"]);
+        assert_eq!(options.reporters, vec!["json", "console"]);
+        assert_eq!(options.ignore_pattern.len(), 1);
+        assert_eq!(options.min_lines, 3);
+        assert_eq!(options.min_tokens, 7);
+        assert_eq!(options.max_lines, 99);
+        assert_eq!(options.max_size_bytes, 2 * 1024);
+        assert_eq!(options.threshold, Some(10.0));
+        assert_eq!(options.mode, super::super::Mode::Weak);
+        assert!(options.silent);
+        assert!(options.absolute);
+        assert!(options.no_symlinks);
+        assert!(options.ignore_case);
+        assert!(!options.gitignore);
+        assert!(options.debug);
+        assert!(options.verbose);
+        assert!(options.skip_local);
+        assert_eq!(options.exit_code, ExitCode::Number(2.0));
+        assert!(options.no_tips);
+    }
+
     #[cfg(unix)]
     #[test]
     fn read_config_preserves_symlink_path_like_upstream() {

package/src/detector/matching/secondary.rs

@@ -329,7 +329,6 @@ mod tests {
             meta: SourceMeta {
                 source_id: source_id.to_string(),
                 format: "javascript".to_string(),
-                content: String::new(),
                 lines: hashes.len(),
                 tokens: hashes.len(),
             },

package/src/detector/model.rs

@@ -1,4 +1,5 @@
 use std::collections::{BTreeMap, HashMap};
+use std::sync::Arc;
 
 use serde::Serialize;
 
@@ -157,7 +158,6 @@ pub(super) struct TokenSpan {
 pub(super) struct SourceMeta {
     pub(super) source_id: String,
     pub(super) format: String,
-    pub(super) content: String,
     pub(super) lines: usize,
     pub(super) tokens: usize,
 }
@@ -185,6 +185,7 @@ pub(super) struct PreparedSource {
 #[derive(Clone, Debug)]
 pub(crate) struct PreparedSourceDraft {
     pub(super) meta: SourceMeta,
-    pub(super) hashes: Vec<u64>,
-    pub(super) spans: Vec<TokenSpan>,
+    pub(super) content: Arc<str>,
+    pub(super) hashes: Arc<Vec<u64>>,
+    pub(super) spans: Arc<Vec<TokenSpan>>,
 }

package/src/detector/prepare.rs

@@ -1,3 +1,5 @@
+use std::sync::Arc;
+
 use rustc_hash::FxHashMap;
 
 use crate::cli::Options;
@@ -26,19 +28,23 @@ pub(super) fn assign_formats(files: &[PreparedSourceDraft]) -> (Vec<FormatId>, V
 }
 
 pub(super) fn prepare_file_maps(file: SourceFile, options: &Options) -> Vec<PreparedSourceDraft> {
-    tokenize_maps_for_detection(&file.content, &file.format, options)
-        .into_iter()
+    let source_id = file.source_id;
+    let content = file.content;
+    let maps = tokenize_maps_for_detection(&content, &file.format, options);
+    let content = Arc::<str>::from(content);
+
+    maps.into_iter()
         .map(|map| {
             let (hashes, spans) = split_tokens(map.tokens);
             let (stat_lines, stat_tokens) = token_stream_statistics(&spans);
             PreparedSourceDraft {
                 meta: SourceMeta {
-                    source_id: file.source_id.clone(),
+                    source_id: source_id.clone(),
                     format: map.format,
-                    content: file.content.clone(),
                     lines: stat_lines,
                     tokens: stat_tokens,
                 },
+                content: Arc::clone(&content),
                 hashes,
                 spans,
             }
@@ -46,7 +52,7 @@ pub(super) fn prepare_file_maps(file: SourceFile, options: &Options) -> Vec<Prep
         .collect()
 }
 
-fn split_tokens(tokens: Vec<DetectionToken>) -> (Vec<u64>, Vec<TokenSpan>) {
+fn split_tokens(tokens: Vec<DetectionToken>) -> (Arc<Vec<u64>>, Arc<Vec<TokenSpan>>) {
     let mut hashes = Vec::with_capacity(tokens.len());
     let mut spans = Vec::with_capacity(tokens.len());
     for token in tokens {
@@ -57,7 +63,7 @@ fn split_tokens(tokens: Vec<DetectionToken>) -> (Vec<u64>, Vec<TokenSpan>) {
             range: token.range,
         });
     }
-    (hashes, spans)
+    (Arc::new(hashes), Arc::new(spans))
 }
 
 fn token_stream_statistics(spans: &[TokenSpan]) -> (usize, usize) {

package/src/detector.rs

@@ -1,4 +1,5 @@
 use std::collections::HashMap;
+use std::sync::Arc;
 
 use rayon::prelude::*;
 use rustc_hash::FxHashSet;
@@ -131,29 +132,36 @@ pub(crate) fn detect_prepared_drafts(
     prepared_drafts: Vec<PreparedSourceDraft>,
     options: &Options,
 ) -> DetectionResult {
+    let include_source_contents = options
+        .reporters
+        .iter()
+        .any(|reporter| matches!(reporter.as_str(), "json" | "xml" | "html" | "consoleFull"));
+    let mut source_contents = HashMap::new();
     let (format_ids, format_names) = assign_formats(&prepared_drafts);
     let prepared_files = prepared_drafts
         .into_iter()
         .enumerate()
-        .map(|(idx, draft)| PreparedSource {
-            meta: draft.meta,
-            stream: TokenStream {
-                source_id: SourceId(idx),
-                format_id: format_ids[idx],
-                hashes: draft.hashes,
-                spans: draft.spans,
-            },
+        .map(|(idx, draft)| {
+            if include_source_contents && !draft.spans.is_empty() {
+                source_contents
+                    .entry(draft.meta.source_id.clone())
+                    .or_insert_with(|| draft.content.to_string());
+            }
+            PreparedSource {
+                meta: draft.meta,
+                stream: TokenStream {
+                    source_id: SourceId(idx),
+                    format_id: format_ids[idx],
+                    hashes: unwrap_or_clone_arc_vec(draft.hashes),
+                    spans: unwrap_or_clone_arc_vec(draft.spans),
+                },
+            }
         })
         .collect::<Vec<_>>();
 
     let mut statistics = Statistics::default();
     let mut sources = Vec::new();
-    let mut source_contents = HashMap::new();
     let mut source_indices_by_format = vec![Vec::new(); format_names.len()];
-    let include_source_contents = options
-        .reporters
-        .iter()
-        .any(|reporter| matches!(reporter.as_str(), "json" | "xml" | "html" | "consoleFull"));
 
     for (idx, prepared) in prepared_files.iter().enumerate() {
         if prepared.stream.spans.is_empty() {
@@ -172,12 +180,6 @@ pub(crate) fn detect_prepared_drafts(
             lines: prepared.meta.lines,
             tokens: prepared.meta.tokens,
         });
-        if include_source_contents {
-            source_contents.insert(
-                prepared.meta.source_id.clone(),
-                prepared.meta.content.clone(),
-            );
-        }
         source_indices_by_format[prepared.stream.format_id.0].push(idx);
     }
 
@@ -222,6 +224,10 @@ fn dedup_exact_clones(clones: &mut Vec<CloneMatch>) {
     clones.retain(|clone| seen.insert(CloneDedupKey::from(clone)));
 }
 
+fn unwrap_or_clone_arc_vec<T: Clone>(value: Arc<Vec<T>>) -> Vec<T> {
+    Arc::try_unwrap(value).unwrap_or_else(|value| (*value).clone())
+}
+
 #[derive(Hash, Eq, PartialEq)]
 struct CloneDedupKey {
     format: String,

package/src/files/discovery.rs

@@ -1,6 +1,10 @@
 use std::fs;
 use std::path::{Path, PathBuf};
-use std::sync::{Arc, Mutex};
+use std::sync::{
+    Arc,
+    atomic::{AtomicBool, Ordering},
+    mpsc,
+};
 
 use anyhow::{Context, Result, anyhow};
 use globset::{Glob, GlobSet, GlobSetBuilder};
@@ -59,14 +63,11 @@ pub fn discover(options: &Options) -> Result<Vec<SourceFile>> {
         let metadata = fs::metadata(root)
             .with_context(|| format!("failed to inspect path `{}`", root.display()))?;
         if metadata.is_file() {
-            collect_candidate(
-                root,
-                root_index,
-                options,
-                &ignore_set,
-                &cwd,
-                &mut candidates,
-            )?;
+            if let Some(candidate) =
+                candidate_for_path(root, root_index, options, &ignore_set, &cwd)?
+            {
+                candidates.push(candidate);
+            }
             continue;
         }
 
@@ -158,14 +159,15 @@ fn collect_candidates_sequential(
         if !context.pattern_set.is_match(relative) {
             continue;
         }
-        collect_candidate(
+        if let Some(candidate) = candidate_for_path(
             path,
             context.root_index,
             context.options,
             context.ignore_set,
             context.cwd,
-            candidates,
-        )?;
+        )? {
+            candidates.push(candidate);
+        }
     }
 
     Ok(())
@@ -176,24 +178,25 @@ fn collect_candidates_parallel(
     context: &CandidateCollectionContext<'_>,
     candidates: &mut Vec<CandidateFile>,
 ) -> Result<()> {
-    let collected = Arc::new(Mutex::new(Vec::new()));
-    let error = Arc::new(Mutex::new(None));
+    let (sender, receiver) = mpsc::channel::<Result<CandidateFile>>();
+    let failed = Arc::new(AtomicBool::new(false));
 
     builder.build_parallel().run(|| {
-        let collected = Arc::clone(&collected);
-        let error = Arc::clone(&error);
+        let sender = sender.clone();
+        let failed = Arc::clone(&failed);
         Box::new(move |entry| {
-            if error.lock().unwrap().is_some() {
+            if failed.load(Ordering::Relaxed) {
                 return WalkState::Quit;
             }
 
             let entry = match entry {
                 Ok(entry) => entry,
                 Err(err) => {
-                    *error.lock().unwrap() = Some(anyhow!(
+                    failed.store(true, Ordering::Relaxed);
+                    let _ = sender.send(Err(anyhow!(
                         "failed to walk path `{}`: {err}",
                         context.root.display()
-                    ));
+                    )));
                     return WalkState::Quit;
                 }
             };
@@ -210,47 +213,50 @@ fn collect_candidates_parallel(
                 return WalkState::Continue;
             }
 
-            let mut local = Vec::with_capacity(1);
-            if let Err(err) = collect_candidate(
+            match candidate_for_path(
                 path,
                 context.root_index,
                 context.options,
                 context.ignore_set,
                 context.cwd,
-                &mut local,
             ) {
-                *error.lock().unwrap() = Some(err);
-                return WalkState::Quit;
-            }
-            if !local.is_empty() {
-                collected.lock().unwrap().extend(local);
+                Ok(Some(candidate)) => {
+                    if sender.send(Ok(candidate)).is_err() {
+                        failed.store(true, Ordering::Relaxed);
+                        return WalkState::Quit;
+                    }
+                }
+                Ok(None) => {}
+                Err(err) => {
+                    failed.store(true, Ordering::Relaxed);
+                    let _ = sender.send(Err(err));
+                    return WalkState::Quit;
+                }
             }
             WalkState::Continue
         })
     });
+    drop(sender);
 
-    if let Some(error) = Arc::try_unwrap(error).unwrap().into_inner().unwrap() {
-        return Err(error);
+    for item in receiver {
+        candidates.push(item?);
     }
 
-    candidates.extend(Arc::try_unwrap(collected).unwrap().into_inner().unwrap());
-
     Ok(())
 }
 
-fn collect_candidate(
+fn candidate_for_path(
     path: &Path,
     root_index: usize,
     options: &Options,
     ignore_set: &IgnoreMatcher,
     cwd: &Path,
-    candidates: &mut Vec<CandidateFile>,
-) -> Result<()> {
+) -> Result<Option<CandidateFile>> {
     if options.no_symlinks && is_symlink(path) {
-        return Ok(());
+        return Ok(None);
     }
     if is_ignored(path, ignore_set, cwd) {
-        return Ok(());
+        return Ok(None);
     }
 
     let format = if let Some(format) =
@@ -269,7 +275,7 @@ fn collect_candidate(
                     options.max_size_bytes
                 );
             }
-            return Ok(());
+            return Ok(None);
         }
         shebang_format_for_path(path, &metadata)?.map(str::to_string)
     };
@@ -277,7 +283,7 @@ fn collect_candidate(
         if options.verbose {
             eprintln!("skipped unsupported format: {}", path.display());
         }
-        return Ok(());
+        return Ok(None);
     };
     if let Some(formats) = &options.formats
         && !formats.contains(format.as_str())
@@ -285,15 +291,13 @@ fn collect_candidate(
         if options.verbose || options.debug {
             println!("{}", format_filter_skip_message(path, &format, cwd));
         }
-        return Ok(());
+        return Ok(None);
     }
-    candidates.push(CandidateFile {
+    Ok(Some(CandidateFile {
         path: path.to_path_buf(),
         format,
         root_index,
-    });
-
-    Ok(())
+    }))
 }
 
 fn read_candidate(
@@ -415,7 +419,7 @@ fn push_pattern_once(
     }
 }
 
-fn normalize_glob_path(path: PathBuf) -> String {
+pub(super) fn normalize_glob_path(path: PathBuf) -> String {
     let mut normalized = PathBuf::new();
     for component in path.components() {
         match component {
@@ -496,7 +500,7 @@ pub(super) fn build_ignore_matcher(patterns: &[String]) -> Result<IgnoreMatcher>
     })
 }
 
-fn build_glob_set(patterns: &[String]) -> Result<GlobSet> {
+pub(super) fn build_glob_set(patterns: &[String]) -> Result<GlobSet> {
     let mut builder = GlobSetBuilder::new();
     if patterns.is_empty() {
         return Ok(builder.build()?);

package/src/files/gitignore.rs

@@ -201,3 +201,83 @@ fn push_gitignore_glob_variants(globs: &mut Vec<String>, path: &Path) {
         globs.push(format!("{relative}/**"));
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::super::test_support::unique_temp_path;
+    use super::*;
+
+    #[test]
+    fn scoped_gitignore_globs_cover_rooted_nested_and_filename_patterns() {
+        let base = std::env::current_dir().unwrap();
+
+        let rooted = scoped_gitignore_globs(&base, "dist", true);
+        assert!(rooted.iter().any(|glob| glob == "dist"));
+        assert!(rooted.iter().any(|glob| glob == "dist/**"));
+
+        let nested = scoped_gitignore_globs(&base, "src/generated", false);
+        assert!(nested.iter().any(|glob| glob == "src/generated"));
+        assert!(nested.iter().any(|glob| glob == "**/src/generated"));
+
+        let filename = scoped_gitignore_globs(&base, "*.snap", false);
+        assert!(filename.iter().any(|glob| glob == "**/*.snap"));
+        assert!(filename.iter().any(|glob| glob == "**/*.snap/**"));
+    }
+
+    #[test]
+    fn gitignore_lines_trim_slashes_backslashes_and_negations() {
+        assert_eq!(
+            gitignore_line_to_globs(r"build\generated", None),
+            vec![
+                "build/generated",
+                "build/generated/**",
+                "**/build/generated",
+                "**/build/generated/**",
+            ]
+        );
+        assert_eq!(
+            gitignore_line_to_globs("!/dist/", None),
+            vec!["!dist", "!dist/**"]
+        );
+    }
+
+    #[test]
+    fn collect_gitignore_patterns_walks_parents_and_repo_exclude_once() {
+        let repo = unique_temp_path("gitignore-repo");
+        let nested = repo.join("packages").join("app");
+        std::fs::create_dir_all(repo.join(".git").join("info")).unwrap();
+        std::fs::create_dir_all(&nested).unwrap();
+        std::fs::write(repo.join(".gitignore"), "/target\n").unwrap();
+        std::fs::write(nested.join(".gitignore"), "local-cache\n").unwrap();
+        std::fs::write(
+            repo.join(".git").join("info").join("exclude"),
+            "repo-only\n",
+        )
+        .unwrap();
+
+        let patterns =
+            collect_gitignore_patterns_with_global(&[nested.clone(), nested.join("src")], None);
+        let _ = std::fs::remove_dir_all(&repo);
+
+        assert!(patterns.iter().any(|pattern| pattern.ends_with("/target")));
+        assert!(
+            patterns
+                .iter()
+                .any(|pattern| pattern.ends_with("/**/local-cache"))
+        );
+        assert!(
+            patterns
+                .iter()
+                .any(|pattern| pattern.ends_with("/**/repo-only"))
+        );
+        let repo_only = patterns
+            .iter()
+            .filter(|pattern| pattern.contains("repo-only"))
+            .collect::<Vec<_>>();
+        let unique_repo_only = repo_only
+            .iter()
+            .copied()
+            .collect::<std::collections::HashSet<_>>();
+        assert_eq!(repo_only.len(), unique_repo_only.len());
+    }
+}