code-gauge 4.4.0 → 4.6.0

package/native/src/duplication.rs

@@ -4,13 +4,14 @@ use std::collections::{HashMap, HashSet};
 use std::sync::OnceLock;
 use tree_sitter::Node;
 
+use crate::near_miss::{Block, Matcher, PairMatch, FILTRATION_PERCENT, MAX_LENGTH_RATIO};
 use crate::types::{
     CrossFileCandidate, CrossFileToken, CrossFileTokenRange, DuplicateBlockOccurrence,
     DuplicationMetrics,
 };
 use crate::util::{all_children, is_identifier_leaf, named_children, node_text, to_int32, Source};
 
-/// Block-like nodes considered as whole-subtree duplicate candidates; see duplication.ts.
+/// Block-like nodes considered as whole-subtree duplicate candidates.
 const DUPLICATE_BLOCK_TYPES: &[&str] = &[
     "statement_block",
     "block",
@@ -314,13 +315,6 @@ impl Default for DuplicationSettings {
         }
     }
 }
-/// N-gram size for the near-miss candidate index (NIL's default); see duplication.ts.
-const NEAR_MISS_NGRAM_SIZE: usize = 5;
-/// Filtration threshold: shared distinct n-grams over the smaller set; see duplication.ts.
-const NEAR_MISS_FILTRATION_PERCENT: usize = 10;
-/// Exclusive bound on shared content-bearing tokens (names and literal values); see duplication.ts.
-const MIN_CONTENT_SIMILARITY_PERCENT: usize = 50;
-
 /// See isLiteralDense in duplication.ts: >= 20% literal values marks a region as data-like.
 fn is_literal_dense(literal_count: usize, token_count: usize) -> bool {
     literal_count * 5 >= token_count
@@ -377,15 +371,17 @@ struct DuplicateCandidate {
     end_line: usize,
 }
 
-/// Detects copy-pasted regions within a file; a faithful port of measureDuplication in
-/// duplication.ts, including its JavaScript int32 hash arithmetic and insertion-order maps.
-pub fn measure_duplication(
-    root: Node<'_>,
-    code_line_numbers: &HashSet<usize>,
-    code: &Source<'_>,
-    settings: &DuplicationSettings,
-) -> DuplicationMetrics {
-    let mut tokens: Vec<Token<'_>> = Vec::new();
+/// A file's normalized token stream with the block and statement structure clone detection
+/// matches over, built once per parse and shared by within-file and cross-file detection.
+pub struct TokenizedSource<'a> {
+    tokens: Vec<Token<'a>>,
+    block_ranges: Vec<TokenRange>,
+    container_statement_ranges: Vec<Vec<TokenRange>>,
+    literal_count_prefix: Vec<usize>,
+}
+
+pub fn tokenize<'a>(root: Node<'_>, code: &Source<'a>) -> TokenizedSource<'a> {
+    let mut tokens: Vec<Token<'a>> = Vec::new();
     let mut block_ranges: Vec<TokenRange> = Vec::new();
     let mut container_statement_ranges: Vec<Vec<TokenRange>> = Vec::new();
     collect_tokens(
@@ -395,66 +391,71 @@ pub fn measure_duplication(
         &mut block_ranges,
         &mut container_statement_ranges,
     );
-
     let literal_count_prefix = build_literal_count_prefix(&tokens);
+    TokenizedSource {
+        tokens,
+        block_ranges,
+        container_statement_ranges,
+        literal_count_prefix,
+    }
+}
+
+/// Detects copy-pasted regions within a file. Fingerprints replicate the JavaScript int32 hash
+/// arithmetic of fingerprintKey in duplication.ts, so its candidates group with the window
+/// candidates cross-file matching fingerprints in TypeScript.
+pub fn measure_duplication(
+    source: &TokenizedSource<'_>,
+    code_line_numbers: &HashSet<usize>,
+    settings: &DuplicationSettings,
+) -> DuplicationMetrics {
+    let tokens = &source.tokens;
+    let literal_count_prefix = &source.literal_count_prefix;
     let mut candidates = collect_block_candidates(
-        &tokens,
-        &literal_count_prefix,
-        &block_ranges,
+        tokens,
+        literal_count_prefix,
+        &source.block_ranges,
         settings.min_tokens,
     );
     candidates.extend(collect_sequence_candidates(
-        &tokens,
-        &literal_count_prefix,
-        &container_statement_ranges,
+        tokens,
+        literal_count_prefix,
+        &source.container_statement_ranges,
         settings.min_tokens,
     ));
     let counted = select_maximal_duplicates(candidates);
     let mut groups = merge_adjacent_groups(to_counted_groups(&counted), settings.max_gap_tokens);
-    let near_miss = collect_near_miss_groups(
-        &tokens,
-        &literal_count_prefix,
-        &block_ranges,
-        settings,
-        &mut groups,
-    );
+    let near_miss = collect_near_miss_groups(source, settings, &mut groups);
     // Near-miss clustering can merge exact groups away, leaving empty entries behind.
     groups.retain(|group| !group.is_empty());
     groups.extend(near_miss);
-    summarize_duplicates(&groups, code_line_numbers, &tokens)
+    summarize_duplicates(&groups, code_line_numbers, tokens)
 }
 
 /// Collects one file's contribution to cross-file clone detection: catalogued candidates (whole
-/// block subtrees plus each statement container's full run) together with the normalized token
-/// stream and statement structure. A faithful port of collectCrossFileDuplicateCandidates in
-/// duplication.ts. Source indexes are emitted in UTF-16 code units (the tree is parsed from
-/// UTF-16, so node byte offsets are halved) to match JavaScript string indexes.
+/// block subtrees plus each statement container's full run), the normalized token stream and
+/// statement structure, and the blocks near-miss comparison considers. Source indexes are emitted
+/// in UTF-16 code units (the tree is parsed from UTF-16, so node byte offsets are halved) to match
+/// JavaScript string indexes.
 pub fn collect_cross_file_file_data(
-    root: Node<'_>,
-    code: &Source<'_>,
+    source: &TokenizedSource<'_>,
     min_tokens: usize,
 ) -> (
     Vec<CrossFileCandidate>,
     Vec<CrossFileToken>,
     Vec<Vec<CrossFileTokenRange>>,
+    Vec<CrossFileTokenRange>,
 ) {
-    let mut tokens: Vec<Token<'_>> = Vec::new();
-    let mut block_ranges: Vec<TokenRange> = Vec::new();
-    let mut container_statement_ranges: Vec<Vec<TokenRange>> = Vec::new();
-    collect_tokens(
-        root,
-        code,
-        &mut tokens,
-        &mut block_ranges,
-        &mut container_statement_ranges,
+    let tokens = &source.tokens;
+    let literal_count_prefix = &source.literal_count_prefix;
+    let mut candidates = collect_block_candidates(
+        tokens,
+        literal_count_prefix,
+        &source.block_ranges,
+        min_tokens,
     );
-    let literal_count_prefix = build_literal_count_prefix(&tokens);
-
-    let mut candidates =
-        collect_block_candidates(&tokens, &literal_count_prefix, &block_ranges, min_tokens);
     // Single-statement containers are catalogued too: a file whose only top-level statement is not
     // a block type (a lone exported table) must still be matchable when wholly copied.
-    for statements in &container_statement_ranges {
+    for statements in &source.container_statement_ranges {
         let (Some(first), Some(last)) = (statements.first(), statements.last()) else {
             continue;
         };
@@ -465,8 +466,8 @@ pub fn collect_cross_file_file_data(
         let fingerprint = format!(
             "s:{}",
             fingerprint_key(
-                &tokens,
-                &literal_count_prefix,
+                tokens,
+                literal_count_prefix,
                 first.start_token_index,
                 last.end_token_index
             )
@@ -507,29 +508,34 @@ pub fn collect_cross_file_file_data(
             end_row: token.end_row,
         })
         .collect();
-    let container_statement_payloads = container_statement_ranges
+    let container_statement_payloads = source
+        .container_statement_ranges
         .iter()
-        .map(|statements| {
-            statements
-                .iter()
-                .map(|range| CrossFileTokenRange {
-                    start_token_index: range.start_token_index,
-                    end_token_index: range.end_token_index,
-                    start_index: range.start_index / 2,
-                    end_index: range.end_index / 2,
-                    start_line: range.start_line,
-                    end_line: range.end_line,
-                })
-                .collect()
-        })
+        .map(|statements| statements.iter().map(to_token_range_payload).collect())
+        .collect();
+    let near_miss_block_payloads = select_near_miss_blocks(source, min_tokens)
+        .into_iter()
+        .map(to_token_range_payload)
         .collect();
     (
         candidate_payloads,
         token_payloads,
         container_statement_payloads,
+        near_miss_block_payloads,
     )
 }
 
+fn to_token_range_payload(range: &TokenRange) -> CrossFileTokenRange {
+    CrossFileTokenRange {
+        start_token_index: range.start_token_index,
+        end_token_index: range.end_token_index,
+        start_index: range.start_index / 2,
+        end_index: range.end_index / 2,
+        start_line: range.start_line,
+        end_line: range.end_line,
+    }
+}
+
 fn collect_tokens<'a>(
     root: Node<'_>,
     code: &Source<'a>,
@@ -577,8 +583,9 @@ fn collect_tokens<'a>(
                     statement_ranges.push(child_range);
                 }
             }
-            // Single-statement containers are recorded too, mirroring collectTokens in
-            // duplication.ts: window enumeration needs two statements and yields nothing for them.
+            // Single-statement containers are recorded too: window enumeration needs two
+            // statements and yields nothing for them, but cross-file matching catalogues each
+            // container's full run.
             if is_container && !statement_ranges.is_empty() {
                 container_statement_ranges.push(statement_ranges);
             }
@@ -731,8 +738,8 @@ fn make_text_token<'a>(
     }
 }
 
-/// The value of a literal as folded into literal-dense fingerprints; see literalValueText in
-/// duplication.ts for the delimiter-independence rationale mirrored here.
+/// The value of a literal as folded into literal-dense fingerprints, independent of its delimiter
+/// spelling (quote style, C# verbatim prefix) so equal values in differently quoted copies match.
 fn literal_value_text<'a>(node: Node<'_>, kind: &str, code: &Source<'a>) -> Cow<'a, str> {
     if kind != "#str" && kind != "#char" {
         return Cow::Borrowed(node_text(node, code));
@@ -759,8 +766,8 @@ fn literal_value_text<'a>(node: Node<'_>, kind: &str, code: &Source<'a>) -> Cow<
     Cow::Borrowed(strip_matching_quotes(text))
 }
 
-/// Strips one matching pair of surrounding ASCII quotes, matching stripMatchingQuotes in
-/// duplication.ts (quote characters are ASCII, so byte indexing is UTF-8 safe).
+/// Strips one matching pair of surrounding ASCII quotes (quote characters are ASCII, so byte
+/// indexing is UTF-8 safe).
 fn strip_matching_quotes(text: &str) -> &str {
     let bytes = text.as_bytes();
     if bytes.len() >= 2 {
@@ -989,7 +996,7 @@ struct ContainerWindows {
     statement_hashes: Vec<i32>,
 }
 
-/// Enumerates runs of consecutive sibling statements; see collectSequenceCandidates in
+/// Enumerates runs of consecutive sibling statements; see collectSequenceWindowCandidates in
 /// duplication.ts for the maximality and sub-window rules replicated here.
 fn collect_sequence_candidates(
     tokens: &[Token<'_>],
@@ -1319,7 +1326,7 @@ fn combine_hashes(hash: i64, value: i64) -> i64 {
     (to_int32(hash).wrapping_mul(31)) as i64 + value
 }
 
-/// Keeps only maximal, non-overlapping duplicates; see selectMaximalDuplicates in duplication.ts.
+/// Keeps only maximal, non-overlapping duplicates; see selectMaximalGroups in duplicateSelection.ts.
 fn select_maximal_duplicates(
     candidates: Vec<DuplicateCandidate>,
 ) -> IndexMap<std::rc::Rc<str>, Vec<DuplicateCandidate>> {
@@ -1474,8 +1481,9 @@ fn merge_adjacent_groups(
     let mut restart = true;
     while restart {
         restart = false;
-        'outer: for left_index in 0..groups.len() {
-            for right_index in left_index + 1..groups.len() {
+        let partners_by_group = collect_gap_adjacent_partners(&groups, max_gap_tokens);
+        'outer: for (left_index, partners) in partners_by_group.iter().enumerate() {
+            for &right_index in partners {
                 let forward =
                     merge_groups(&groups[left_index], &groups[right_index], max_gap_tokens);
                 let swapped = forward.is_none();
@@ -1516,6 +1524,43 @@ fn merge_adjacent_groups(
     groups
 }
 
+/// Per group index, the ascending indexes of later groups that merge_groups can pair with it;
+/// see collectGapAdjacentPartners in duplication.ts.
+fn collect_gap_adjacent_partners(
+    groups: &[Vec<CountedOccurrence>],
+    max_gap_tokens: usize,
+) -> Vec<Vec<usize>> {
+    let mut starts: Vec<(usize, usize)> = groups
+        .iter()
+        .enumerate()
+        .flat_map(|(group_index, group)| {
+            group
+                .iter()
+                .map(move |occurrence| (occurrence.start_token_index, group_index))
+        })
+        .collect();
+    starts.sort_unstable();
+    let mut partners: Vec<std::collections::BTreeSet<usize>> =
+        vec![std::collections::BTreeSet::new(); groups.len()];
+    for (group_index, group) in groups.iter().enumerate() {
+        for occurrence in group {
+            let first = starts.partition_point(|&(start, _)| start < occurrence.end_token_index);
+            for &(start, other) in &starts[first..] {
+                if start > occurrence.end_token_index + max_gap_tokens {
+                    break;
+                }
+                if other != group_index {
+                    partners[other.min(group_index)].insert(other.max(group_index));
+                }
+            }
+        }
+    }
+    partners
+        .into_iter()
+        .map(|set| set.into_iter().collect())
+        .collect()
+}
+
 fn group_sort_key(group: &[CountedOccurrence]) -> (usize, usize) {
     group
         .first()
@@ -1611,79 +1656,136 @@ fn merge_groups(
     })
 }
 
+/// A verified near-miss pair as (block, core, block, core); a `None` core is a whole-block match.
+type MatchEdge = (usize, Option<(usize, usize)>, usize, Option<(usize, usize)>);
+
 /// Detects near-miss (Type-3) clone groups among block candidates the exact pipeline left
-/// unreported; a faithful port of collectNearMissGroups in duplication.ts (NIL-style n-gram
-/// filtration, then token-level LCS with NiCad-style per-fragment similarity, then transitive
-/// clustering of verified pairs).
+/// unreported: NIL-style n-gram filtration, then pair verification (near_miss::Matcher), then
+/// transitive clustering of verified pairs (crossFileNearMiss.ts applies the same model across
+/// files). A block that matched only locally is reported as its matched cores (overlapping cores
+/// merged), each clustered with its own partners, so code no verified pair matched never counts
+/// as duplicated.
 fn collect_near_miss_groups(
-    tokens: &[Token<'_>],
-    literal_count_prefix: &[usize],
-    block_ranges: &[TokenRange],
+    source: &TokenizedSource<'_>,
     settings: &DuplicationSettings,
     reported_groups: &mut [Vec<CountedOccurrence>],
 ) -> Vec<Vec<CountedOccurrence>> {
     if settings.min_similarity_percent >= 100 {
         return Vec::new();
     }
-    let mut eligible: Vec<&TokenRange> = block_ranges
-        .iter()
-        .filter(|range| {
-            let token_count = range.end_token_index - range.start_token_index;
-            let literal_count = literal_count_prefix
-                .get(range.end_token_index)
-                .copied()
-                .unwrap_or(0)
-                - literal_count_prefix
-                    .get(range.start_token_index)
-                    .copied()
-                    .unwrap_or(0);
-            token_count >= settings.min_tokens && !is_literal_dense(literal_count, token_count)
-        })
-        .collect();
-    eligible.sort_by_key(|range| {
-        (
-            range.start_token_index,
-            std::cmp::Reverse(range.end_token_index),
-        )
-    });
-    let comparable = select_comparable_blocks(&eligible);
+    let tokens = &source.tokens;
+    let comparable = select_near_miss_blocks(source, settings.min_tokens);
     if comparable.len() < 2 {
         return Vec::new();
     }
 
-    // Reported-group indices whose occurrences overlap each comparable block: such blocks anchor
-    // near-miss comparisons but are never re-reported.
-    let touched_groups_by_block: Vec<Vec<usize>> = comparable
+    // Reported-group indices whose occurrences overlap a token range: near-miss nodes covering
+    // such content anchor comparisons but are never re-reported.
+    let touched_groups_in = |start: usize, end: usize| -> Vec<usize> {
+        reported_groups
+            .iter()
+            .enumerate()
+            .filter(|(_, group)| {
+                group.iter().any(|occurrence| {
+                    occurrence.start_token_index < end && start < occurrence.end_token_index
+                })
+            })
+            .map(|(group_index, _)| group_index)
+            .collect()
+    };
+
+    let (symbols, is_content) = to_symbol_stream(tokens);
+    let statements = top_level_statement_finder(&source.container_statement_ranges);
+    let mut blocks: Vec<Block> = comparable
         .iter()
         .map(|range| {
-            reported_groups
-                .iter()
-                .enumerate()
-                .filter(|(_, group)| {
-                    group.iter().any(|occurrence| {
-                        occurrence.start_token_index < range.end_token_index
-                            && range.start_token_index < occurrence.end_token_index
-                    })
-                })
-                .map(|(group_index, _)| group_index)
-                .collect()
+            Block::new(
+                &symbols,
+                &is_content,
+                range.start_token_index,
+                range.end_token_index,
+                statements(range.start_token_index, range.end_token_index),
+            )
         })
         .collect();
+    let matcher = Matcher::new(
+        &mut blocks,
+        settings.min_tokens,
+        settings.min_similarity_percent,
+    );
 
-    // Interned per call so a file's symbol ids (and thus its n-gram hashes) never depend on which
-    // other files the process measured before it.
-    let mut symbol_id_by_token_hashes: HashMap<(i32, i32, i32, i32), i32> = HashMap::new();
-    let sequences: Vec<NormalizedBlock> = comparable
-        .iter()
-        .map(|range| normalize_block_sequence(tokens, range, &mut symbol_id_by_token_hashes))
-        .collect();
-    let ngram_sets: Vec<HashSet<i32>> = sequences
+    let block_touched: Vec<bool> = comparable
         .iter()
-        .map(|block| collect_ngram_set(&block.sequence))
+        .map(|range| !touched_groups_in(range.start_token_index, range.end_token_index).is_empty())
         .collect();
-    let shared_counts = count_shared_ngrams(&ngram_sets);
+    let mut edges: Vec<MatchEdge> = Vec::new();
+    for_each_candidate_pair(
+        &blocks,
+        settings.min_similarity_percent,
+        |left_index, right_index| match matcher.verify(&blocks[left_index], &blocks[right_index]) {
+            None => {}
+            // A whole match between two blocks that both overlap reported content could never join
+            // a group, and recording it would collapse the blocks' core nodes.
+            Some(PairMatch::Whole)
+                if !(block_touched[left_index] && block_touched[right_index]) =>
+            {
+                edges.push((left_index, None, right_index, None))
+            }
+            Some(PairMatch::Whole) => {}
+            Some(PairMatch::Local(cores)) => {
+                for (left_core, right_core) in cores {
+                    edges.push((left_index, Some(left_core), right_index, Some(right_core)));
+                }
+            }
+        },
+    );
+
+    // Clustering runs over (block, core) nodes: a block with a recorded whole match is one
+    // node, and otherwise each union of its overlapping local cores is its own node, so disjoint
+    // cores matched with different partners fall into separate groups.
+    let mut matched_whole = vec![false; comparable.len()];
+    let mut local_cores: Vec<Vec<(usize, usize)>> = vec![Vec::new(); comparable.len()];
+    for &(left_index, left_core, right_index, right_core) in &edges {
+        for (index, core) in [(left_index, left_core), (right_index, right_core)] {
+            match core {
+                Some(core) => local_cores[index].push(core),
+                None => matched_whole[index] = true,
+            }
+        }
+    }
+    let mut node_blocks: Vec<usize> = Vec::new();
+    let mut node_spans: Vec<Option<(usize, usize)>> = Vec::new();
+    let mut first_node_by_block: Vec<usize> = Vec::with_capacity(comparable.len());
+    for index in 0..comparable.len() {
+        first_node_by_block.push(node_blocks.len());
+        let cores = if matched_whole[index] {
+            Vec::new()
+        } else {
+            merge_overlapping_cores(&local_cores[index])
+        };
+        if cores.is_empty() {
+            node_blocks.push(index);
+            node_spans.push(None);
+        }
+        for core in cores {
+            node_blocks.push(index);
+            node_spans.push(Some(core));
+        }
+    }
+    let node_of = |index: usize, core: Option<(usize, usize)>| {
+        let first = first_node_by_block[index];
+        match core {
+            Some(core) if !matched_whole[index] => (first..node_blocks.len())
+                .take_while(|&node| node_blocks[node] == index)
+                .find(|&node| {
+                    node_spans[node].is_some_and(|span| span.0 <= core.0 && core.1 <= span.1)
+                })
+                .expect("every local core lies in one of its block's merged cores"),
+            _ => first,
+        }
+    };
 
-    let mut parent: Vec<usize> = (0..comparable.len()).collect();
+    let mut parent: Vec<usize> = (0..node_blocks.len()).collect();
     fn find(parent: &mut [usize], mut index: usize) -> usize {
         let mut root = index;
         while parent[root] != root {
@@ -1696,52 +1798,84 @@ fn collect_near_miss_groups(
         }
         root
     }
-    for (&(left_index, right_index), &shared) in &shared_counts {
-        let left = &sequences[left_index];
-        let right = &sequences[right_index];
-        // Two already-reported blocks have nothing new to contribute to each other.
-        if !touched_groups_by_block[left_index].is_empty()
-            && !touched_groups_by_block[right_index].is_empty()
-        {
-            continue;
-        }
-        let min_ngrams = ngram_sets[left_index]
-            .len()
-            .min(ngram_sets[right_index].len());
-        if shared * 100 < NEAR_MISS_FILTRATION_PERCENT * min_ngrams {
-            continue;
-        }
-        // A structural match must be backed by shared content (names and literal values); the
-        // bound is exclusive, matching duplication.ts.
-        if content_overlap(left, right) * 100
-            <= MIN_CONTENT_SIMILARITY_PERCENT * left.content_total.max(right.content_total)
+    // Coverage is judged per node: a core is covered only when a reported occurrence overlaps
+    // the core itself, not merely elsewhere in its block.
+    let node_range = |node: usize| {
+        node_spans[node].unwrap_or((
+            comparable[node_blocks[node]].start_token_index,
+            comparable[node_blocks[node]].end_token_index,
+        ))
+    };
+    let touched_groups_by_node: Vec<Vec<usize>> = (0..node_blocks.len())
+        .map(|node| {
+            let (start, end) = node_range(node);
+            touched_groups_in(start, end)
+        })
+        .collect();
+    for &(left_index, left_core, right_index, right_core) in &edges {
+        let (left_node, right_node) = (
+            node_of(left_index, left_core),
+            node_of(right_index, right_core),
+        );
+        // Two already-reported nodes have nothing new to contribute to each other.
+        if !touched_groups_by_node[left_node].is_empty()
+            && !touched_groups_by_node[right_node].is_empty()
         {
             continue;
         }
-        // Per-fragment similarity against the larger block (NiCad semantics).
-        if lcs_length(&left.sequence, &right.sequence) * 100
-            >= settings.min_similarity_percent * left.sequence.len().max(right.sequence.len())
-        {
-            let left_root = find(&mut parent, left_index);
-            let right_root = find(&mut parent, right_index);
-            parent[left_root.max(right_root)] = left_root.min(right_root);
-        }
+        let left_root = find(&mut parent, left_node);
+        let right_root = find(&mut parent, right_node);
+        parent[left_root.max(right_root)] = left_root.min(right_root);
     }
 
     let mut members_by_root: IndexMap<usize, Vec<usize>> = IndexMap::new();
-    for index in 0..comparable.len() {
-        let root = find(&mut parent, index);
-        members_by_root.entry(root).or_default().push(index);
-    }
-    let to_occurrence = |range: &TokenRange| CountedOccurrence {
-        shared_with_merged_group: false,
-        segments: vec![(range.start_token_index, range.end_token_index)],
-        token_count: range.end_token_index - range.start_token_index,
-        start_token_index: range.start_token_index,
-        end_token_index: range.end_token_index,
-        start_line: range.start_line,
-        end_line: range.end_line,
+    for node in 0..node_blocks.len() {
+        let root = find(&mut parent, node);
+        members_by_root.entry(root).or_default().push(node);
+    }
+    // A group's nodes from one block become ONE occurrence whose segments are its cores, so the
+    // fragment-weighted count charges the block as one copy (as for gapped clones), not once per
+    // core.
+    let to_occurrences = |nodes: &[usize]| -> Vec<CountedOccurrence> {
+        let mut spans_by_block: IndexMap<usize, Vec<Option<(usize, usize)>>> = IndexMap::new();
+        for &node in nodes {
+            spans_by_block
+                .entry(node_blocks[node])
+                .or_default()
+                .push(node_spans[node]);
+        }
+        spans_by_block
+            .into_iter()
+            .map(|(block, spans)| {
+                let range = comparable[block];
+                let mut segments: Vec<(usize, usize)> = spans
+                    .iter()
+                    .map(|span| span.unwrap_or((range.start_token_index, range.end_token_index)))
+                    .collect();
+                segments.sort_unstable();
+                let whole = spans.iter().any(Option::is_none);
+                let (start, end) = (segments[0].0, segments[segments.len() - 1].1);
+                CountedOccurrence {
+                    shared_with_merged_group: false,
+                    token_count: segments.iter().map(|segment| segment.1 - segment.0).sum(),
+                    segments,
+                    start_token_index: start,
+                    end_token_index: end,
+                    start_line: if whole {
+                        range.start_line
+                    } else {
+                        tokens[start].start_row + 1
+                    },
+                    end_line: if whole {
+                        range.end_line
+                    } else {
+                        tokens[end - 1].end_row + 1
+                    },
+                }
+            })
+            .collect()
     };
+    let touched_groups_of = |node: usize| &touched_groups_by_node[node];
     let mut groups: Vec<Vec<CountedOccurrence>> = Vec::new();
     for members in members_by_root.values() {
         if members.len() < 2 {
@@ -1750,38 +1884,36 @@ fn collect_near_miss_groups(
         let uncovered: Vec<usize> = members
             .iter()
             .copied()
-            .filter(|&index| touched_groups_by_block[index].is_empty())
+            .filter(|&index| touched_groups_of(index).is_empty())
             .collect();
         let covered: Vec<usize> = members
             .iter()
             .copied()
-            .filter(|&index| !touched_groups_by_block[index].is_empty())
+            .filter(|&index| !touched_groups_of(index).is_empty())
             .collect();
         if covered.is_empty() {
-            groups.push(
-                members
-                    .iter()
-                    .map(|&index| to_occurrence(comparable[index]))
-                    .collect(),
-            );
+            let occurrences = to_occurrences(members);
+            if occurrences.len() >= 2 {
+                groups.push(occurrences);
+            }
             continue;
         }
         if uncovered.is_empty() {
             continue;
         }
-        // An anchored cluster extends a reported group only when that group lies entirely inside
-        // the cluster; see collectNearMissGroups in duplication.ts.
+        // An anchored cluster extends a reported group only when every occurrence of that group
+        // overlaps one of the cluster's member nodes: an occurrence disjoint from all members
+        // reports content the cluster does not share.
         let overlaps_member = |occurrence: &CountedOccurrence| {
             members.iter().any(|&index| {
-                let range = comparable[index];
-                occurrence.start_token_index < range.end_token_index
-                    && range.start_token_index < occurrence.end_token_index
+                let (start, end) = node_range(index);
+                occurrence.start_token_index < end && start < occurrence.end_token_index
             })
         };
         // Ascending by construction: BTreeSet iteration is sorted and filter preserves order.
         let fully_clustered: Vec<usize> = covered
             .iter()
-            .flat_map(|&index| touched_groups_by_block[index].iter().copied())
+            .flat_map(|&index| touched_groups_of(index).iter().copied())
             .collect::<std::collections::BTreeSet<usize>>()
             .into_iter()
             .filter(|&group_index| {
@@ -1790,33 +1922,57 @@ fn collect_near_miss_groups(
             })
             .collect();
         if let Some((&target_index, source_indexes)) = fully_clustered.split_first() {
-            // Rebuild the component as ONE group with one coalesced occurrence per member block;
-            // see collectNearMissGroups in duplication.ts.
+            // Rebuild the component as ONE group with one coalesced occurrence per member block:
+            // the fragments every node of a block overlaps are collected together, since a block's
+            // cores are parts of one copy.
             let mut consumed: HashSet<(usize, usize)> = HashSet::new();
             let mut merged: Vec<CountedOccurrence> = Vec::new();
+            let mut unanchored_nodes: Vec<usize> = Vec::new();
+            let mut nodes_by_block: IndexMap<usize, Vec<usize>> = IndexMap::new();
             for &member_index in members {
-                let range = comparable[member_index];
+                nodes_by_block
+                    .entry(node_blocks[member_index])
+                    .or_default()
+                    .push(member_index);
+            }
+            for block_nodes in nodes_by_block.values() {
                 // Occurrences of ONE group are distinct copies; only fragments from DIFFERENT
                 // groups belong to the same copy. Consecutive position-order slices keep the
-                // coalesced spans disjoint; see collectNearMissGroups in duplication.ts.
+                // coalesced spans disjoint.
                 let mut fragments: Vec<(CountedOccurrence, usize)> = Vec::new();
-                for &group_index in &fully_clustered {
-                    for (occurrence_index, occurrence) in
-                        reported_groups[group_index].iter().enumerate()
-                    {
-                        if !consumed.contains(&(group_index, occurrence_index))
-                            && occurrence.start_token_index < range.end_token_index
-                            && range.start_token_index < occurrence.end_token_index
+                let mut plain_nodes: Vec<usize> = Vec::new();
+                for &node in block_nodes {
+                    let (range_start, range_end) = node_range(node);
+                    let fragment_count = fragments.len();
+                    for &group_index in &fully_clustered {
+                        for (occurrence_index, occurrence) in
+                            reported_groups[group_index].iter().enumerate()
                         {
-                            consumed.insert((group_index, occurrence_index));
-                            fragments.push((occurrence.clone(), group_index));
+                            if !consumed.contains(&(group_index, occurrence_index))
+                                && occurrence.start_token_index < range_end
+                                && range_start < occurrence.end_token_index
+                            {
+                                consumed.insert((group_index, occurrence_index));
+                                fragments.push((occurrence.clone(), group_index));
+                            }
                         }
                     }
+                    if fragments.len() == fragment_count && touched_groups_of(node).is_empty() {
+                        plain_nodes.push(node);
+                    }
+                }
+                if fragments.is_empty() {
+                    unanchored_nodes.extend(plain_nodes);
+                } else {
+                    // An untouched core of a block that also holds fragments is part of the same
+                    // copy; a group index no reported group uses keeps it in that copy.
+                    for occurrence in to_occurrences(&plain_nodes) {
+                        fragments.push((occurrence, usize::MAX));
+                    }
                 }
                 fragments.sort_by_key(|(occurrence, _)| {
                     (occurrence.start_token_index, occurrence.end_token_index)
                 });
-                let had_fragments = !fragments.is_empty();
                 let mut copy_parts: Vec<CountedOccurrence> = Vec::new();
                 let mut copy_groups: HashSet<usize> = HashSet::new();
                 for (occurrence, group_index) in fragments {
@@ -1830,10 +1986,8 @@ fn collect_near_miss_groups(
                 if !copy_parts.is_empty() {
                     merged.push(coalesce_occurrences(copy_parts));
                 }
-                if !had_fragments && touched_groups_by_block[member_index].is_empty() {
-                    merged.push(to_occurrence(comparable[member_index]));
-                }
             }
+            merged.extend(to_occurrences(&unanchored_nodes));
             merged.sort_by_key(|occurrence| {
                 (occurrence.start_token_index, occurrence.end_token_index)
             });
@@ -1846,22 +2000,46 @@ fn collect_near_miss_groups(
             for &source_index in source_indexes {
                 reported_groups[source_index].clear();
             }
-        } else if uncovered.len() >= 2 {
-            groups.push(
-                uncovered
-                    .iter()
-                    .map(|&index| to_occurrence(comparable[index]))
-                    .collect(),
-            );
+        } else {
+            let occurrences = to_occurrences(&uncovered);
+            if occurrences.len() >= 2 {
+                groups.push(occurrences);
+            }
         }
     }
     groups.sort_by_key(|group| group_sort_key(group));
     groups
 }
 
-/// Keeps the block ranges the near-miss phase compares; a faithful port of selectComparableBlocks
-/// in duplication.ts (wrappers whose subtree branches into two or more disjoint eligible
-/// sub-blocks are descended through; linear chains keep their top).
+/// The mutually disjoint blocks near-miss comparison considers: at least `min_tokens` long, not
+/// literal-dense (data tables are compared by value, not shape), and selected by
+/// select_comparable_blocks.
+fn select_near_miss_blocks<'s>(
+    source: &'s TokenizedSource<'_>,
+    min_tokens: usize,
+) -> Vec<&'s TokenRange> {
+    let literal_count_prefix = &source.literal_count_prefix;
+    let mut eligible: Vec<&TokenRange> = source
+        .block_ranges
+        .iter()
+        .filter(|range| {
+            let token_count = range.end_token_index - range.start_token_index;
+            let literal_count = literal_count_prefix[range.end_token_index]
+                - literal_count_prefix[range.start_token_index];
+            token_count >= min_tokens && !is_literal_dense(literal_count, token_count)
+        })
+        .collect();
+    eligible.sort_by_key(|range| {
+        (
+            range.start_token_index,
+            std::cmp::Reverse(range.end_token_index),
+        )
+    });
+    select_comparable_blocks(&eligible)
+}
+
+/// Keeps the block ranges the near-miss phase compares: wrappers whose subtree branches into two
+/// or more disjoint eligible sub-blocks are descended through; linear chains keep their top.
 fn select_comparable_blocks<'a>(eligible: &[&'a TokenRange]) -> Vec<&'a TokenRange> {
     struct ForestNode<'a> {
         range: &'a TokenRange,
@@ -1919,8 +2097,7 @@ fn select_comparable_blocks<'a>(eligible: &[&'a TokenRange]) -> Vec<&'a TokenRan
     kept
 }
 
-/// One copy's fragments (an exact prefix and suffix split by a large edit) as one occurrence;
-/// mirrors coalesceOccurrences in duplication.ts.
+/// One copy's fragments (an exact prefix and suffix split by a large edit) as one occurrence.
 fn coalesce_occurrences(occurrences: Vec<CountedOccurrence>) -> CountedOccurrence {
     if occurrences.len() == 1 {
         return occurrences.into_iter().next().expect("non-empty");
@@ -1959,149 +2136,125 @@ fn coalesce_occurrences(occurrences: Vec<CountedOccurrence>) -> CountedOccurrenc
     }
 }
 
-struct NormalizedBlock {
-    sequence: Vec<i32>,
-    /// Occurrences per content-bearing symbol (names and literal values), for the content gate.
-    content_count_by_symbol: HashMap<i32, usize>,
-    content_total: usize,
+/// The unions of overlapping cores, in position order.
+fn merge_overlapping_cores(cores: &[(usize, usize)]) -> Vec<(usize, usize)> {
+    let mut sorted = cores.to_vec();
+    sorted.sort_unstable();
+    let mut merged: Vec<(usize, usize)> = Vec::new();
+    for (start, end) in sorted {
+        match merged.last_mut() {
+            Some(last) if start < last.1 => last.1 = last.1.max(end),
+            _ => merged.push((start, end)),
+        }
+    }
+    merged
 }
 
-/// A block's tokens as comparable integers; see normalizeBlockSequence in duplication.ts
-/// (literal VALUES are folded into the symbol, unlike the exact fingerprint's kind tags).
-fn normalize_block_sequence(
-    tokens: &[Token<'_>],
-    range: &TokenRange,
-    symbol_id_by_token_hashes: &mut HashMap<(i32, i32, i32, i32), i32>,
-) -> NormalizedBlock {
-    let mut sequence = Vec::with_capacity(range.end_token_index - range.start_token_index);
-    let mut index_by_identifier: HashMap<&str, i32> = HashMap::new();
-    let mut content_count_by_symbol: HashMap<i32, usize> = HashMap::new();
-    let mut content_total = 0usize;
-    for token in &tokens[range.start_token_index..range.end_token_index.min(tokens.len())] {
-        let value = if token.is_id {
-            let next_index = index_by_identifier.len() as i32;
-            let identifier_index = *index_by_identifier
-                .entry(token.text.as_ref())
-                .or_insert(next_index);
-            -(identifier_index + 1)
-        } else {
-            let next_id = symbol_id_by_token_hashes.len() as i32;
-            let id = *symbol_id_by_token_hashes
+/// The file's tokens as a near-miss symbol stream: identifiers as -(file-level id + 1), every
+/// other token interned from its hash pairs (literal VALUES folded in, unlike the exact
+/// fingerprint's kind tags), plus which tokens are content-bearing (names and literal values).
+/// Interned per call so a file's symbol ids (and thus its n-gram hashes) never depend on which
+/// other files the process measured before it.
+fn to_symbol_stream(tokens: &[Token<'_>]) -> (Vec<i32>, Vec<bool>) {
+    let mut symbol_by_token_hashes: HashMap<(i32, i32, i32, i32), i32> = HashMap::new();
+    let mut id_by_identifier: HashMap<&str, i32> = HashMap::new();
+    tokens
+        .iter()
+        .map(|token| {
+            if token.is_id {
+                let next_id = id_by_identifier.len() as i32;
+                let id = *id_by_identifier
+                    .entry(token.text.as_ref())
+                    .or_insert(next_id);
+                return (-(id + 1), false);
+            }
+            let next_symbol = symbol_by_token_hashes.len() as i32;
+            let symbol = *symbol_by_token_hashes
                 .entry((
                     token.text_hash,
                     token.text_hash2,
                     token.literal_hash.unwrap_or(0),
                     token.literal_hash2.unwrap_or(0),
                 ))
-                .or_insert(next_id);
-            if token.is_name || token.literal_hash.is_some() {
-                *content_count_by_symbol.entry(id).or_insert(0) += 1;
-                content_total += 1;
-            }
-            id
-        };
-        sequence.push(value);
-    }
-    NormalizedBlock {
-        sequence,
-        content_count_by_symbol,
-        content_total,
-    }
-}
-
-/// Multiset overlap of two blocks' content-bearing symbols, for the content gate.
-fn content_overlap(left: &NormalizedBlock, right: &NormalizedBlock) -> usize {
-    let (smaller, larger) =
-        if left.content_count_by_symbol.len() <= right.content_count_by_symbol.len() {
-            (left, right)
-        } else {
-            (right, left)
-        };
-    smaller
-        .content_count_by_symbol
-        .iter()
-        .map(|(symbol, count)| {
-            (*count).min(
-                larger
-                    .content_count_by_symbol
-                    .get(symbol)
-                    .copied()
-                    .unwrap_or(0),
-            )
+                .or_insert(next_symbol);
+            (symbol, token.is_name || token.literal_hash.is_some())
         })
-        .sum()
+        .unzip()
 }
 
-/// The distinct 5-gram hashes of a normalized block sequence, matching collectNgramSet exactly.
-fn collect_ngram_set(sequence: &[i32]) -> HashSet<i32> {
-    if sequence.len() < NEAR_MISS_NGRAM_SIZE {
-        return HashSet::new();
-    }
-    // Exact upper bound: one n-gram per window, and most windows hash distinctly.
-    let mut ngrams = HashSet::with_capacity(sequence.len() - NEAR_MISS_NGRAM_SIZE + 1);
-    for window in sequence.windows(NEAR_MISS_NGRAM_SIZE) {
-        let mut hash: i32 = 5381;
-        for &value in window {
-            hash = hash.wrapping_mul(31).wrapping_add(value);
+/// Returns a lookup of the outermost container statements inside a token range, excluding a
+/// statement spanning the whole range (the block itself).
+fn top_level_statement_finder(
+    container_statement_ranges: &[Vec<TokenRange>],
+) -> impl Fn(usize, usize) -> Vec<(usize, usize)> {
+    let mut statements: Vec<(usize, usize)> = container_statement_ranges
+        .iter()
+        .flatten()
+        .map(|range| (range.start_token_index, range.end_token_index))
+        .filter(|(start, end)| start < end)
+        .collect();
+    statements.sort_by_key(|&(start, end)| (start, std::cmp::Reverse(end)));
+    move |start, end| {
+        let mut top_level: Vec<(usize, usize)> = Vec::new();
+        let first = statements.partition_point(|statement| statement.0 < start);
+        for &statement in statements[first..]
+            .iter()
+            .take_while(|statement| statement.0 < end)
+        {
+            let nested = top_level.last().is_some_and(|last| statement.0 < last.1);
+            if statement.1 <= end && statement != (start, end) && !nested {
+                top_level.push(statement);
+            }
         }
-        ngrams.insert(hash);
+        top_level
     }
-    ngrams
 }
 
-/// Shared distinct-n-gram counts per block pair (left < right).
-fn count_shared_ngrams(ngram_sets: &[HashSet<i32>]) -> HashMap<(usize, usize), usize> {
-    let mut blocks_by_ngram: HashMap<i32, Vec<usize>> = HashMap::new();
-    for (block_index, ngrams) in ngram_sets.iter().enumerate() {
-        for &ngram in ngrams {
-            blocks_by_ngram.entry(ngram).or_default().push(block_index);
-        }
-    }
-    let mut shared_counts: HashMap<(usize, usize), usize> = HashMap::new();
-    for blocks in blocks_by_ngram.values() {
-        for (position, &left_index) in blocks.iter().enumerate() {
-            // Bucket indices are appended in ascending block order, so left < right already.
-            for &right_index in &blocks[position + 1..] {
-                *shared_counts.entry((left_index, right_index)).or_insert(0) += 1;
+/// Visits every block pair sharing at least FILTRATION_PERCENT of the smaller block's distinct
+/// n-grams, except pairs whose length ratio rules out both whole-block similarity and
+/// MAX_LENGTH_RATIO; the same scan as forEachCandidatePair in crossFileNearMiss.ts. Blocks are
+/// visited in ascending length, so each posting list is scanned backwards only while its blocks are
+/// long enough, and shared counts accumulate in a dense counter instead of a pair map.
+fn for_each_candidate_pair(
+    blocks: &[Block],
+    min_similarity_percent: usize,
+    mut visit: impl FnMut(usize, usize),
+) {
+    let mut order: Vec<usize> = (0..blocks.len()).collect();
+    order.sort_by_key(|&index| blocks[index].len());
+    let mut postings: HashMap<i32, Vec<usize>> = HashMap::new();
+    let mut shared_counts = vec![0usize; blocks.len()];
+    let mut touched: Vec<usize> = Vec::new();
+    for right in order {
+        let length = blocks[right].len();
+        let min_left_length = length
+            .div_ceil(MAX_LENGTH_RATIO)
+            .min((min_similarity_percent * length).div_ceil(100));
+        for &ngram in &blocks[right].ngrams {
+            let posting = postings.entry(ngram).or_default();
+            for &left in posting.iter().rev() {
+                if blocks[left].len() < min_left_length {
+                    break;
+                }
+                if shared_counts[left] == 0 {
+                    touched.push(left);
+                }
+                shared_counts[left] += 1;
             }
+            posting.push(right);
         }
-    }
-    shared_counts
-}
-
-/// Longest-common-subsequence LENGTH via the Allison–Dix bit-parallel recurrence. Only the length
-/// is needed and LCS length is algorithm-independent, so u64 words are safe even though the
-/// TypeScript port in src/duplication.ts uses 32-bit words.
-fn lcs_length(a: &[i32], b: &[i32]) -> usize {
-    if a.is_empty() || b.is_empty() {
-        return 0;
-    }
-    let word_count = a.len().div_ceil(64);
-    let mut position_masks: HashMap<i32, Vec<u64>> = HashMap::new();
-    for (index, &symbol) in a.iter().enumerate() {
-        position_masks
-            .entry(symbol)
-            .or_insert_with(|| vec![0; word_count])[index / 64] |= 1u64 << (index % 64);
-    }
-
-    let mut v = vec![0u64; word_count];
-    for symbol in b {
-        let match_mask = position_masks.get(symbol);
-        // `(v << 1) | 1` shifts a carry bit across words; subtraction borrows across words.
-        let mut shift_carry = 1u64;
-        let mut borrow = 0u64;
-        for (word, slot) in v.iter_mut().enumerate() {
-            let previous = *slot;
-            let x = match_mask.map_or(0, |mask| mask[word]) | previous;
-            let shifted = (previous << 1) | shift_carry;
-            shift_carry = previous >> 63;
-            let (partial, underflow1) = x.overflowing_sub(shifted);
-            let (difference, underflow2) = partial.overflowing_sub(borrow);
-            borrow = u64::from(underflow1 || underflow2);
-            *slot = x & !difference;
+        // Ascending so the visit order does not depend on the n-gram set's iteration order.
+        touched.sort_unstable();
+        for &left in &touched {
+            let shared = std::mem::take(&mut shared_counts[left]);
+            if shared * 100
+                >= FILTRATION_PERCENT * blocks[left].ngrams.len().min(blocks[right].ngrams.len())
+            {
+                visit(left, right);
+            }
         }
+        touched.clear();
     }
-    v.iter().map(|word| word.count_ones() as usize).sum()
 }
 
 /// Redundant copies one group adds to duplicate_block_count; a faithful port of