code-gauge 10.1.0 → 10.2.1

package/native/src/duplication.rs

@@ -4,7 +4,9 @@ use std::borrow::Cow;
 use std::sync::OnceLock;
 use tree_sitter::Node;
 
-use crate::near_miss::{Block, Matcher, PairMatch, FILTRATION_PERCENT, MAX_LENGTH_RATIO};
+use crate::near_miss::{
+    Block, CorePair, Matcher, PairMatch, TokenRuns, FILTRATION_PERCENT, MAX_LENGTH_RATIO,
+};
 use crate::tree_index::NodeExt;
 use crate::types::{
     CrossFileCandidate, CrossFileToken, CrossFileTokenRange, DuplicateBlockOccurrence,
@@ -1577,9 +1579,16 @@ fn dedupe_by_region(group: Vec<DuplicateCandidate>) -> Vec<DuplicateCandidate> {
 #[derive(Clone)]
 struct CountedOccurrence {
     segments: Vec<(usize, usize)>,
-    /// Set on a retained group's occurrences that a partial gapped merge also paired into a
-    /// merged group: their spans are counted there, so block counting must not count them again.
+    /// Set on an occurrence whose span another group counts, so block counting must not count it
+    /// again: a retained group's occurrence that a partial gapped merge also paired into a merged
+    /// group, and a near-miss anchor, which overlaps the reported occurrence it was matched through.
     shared_with_merged_group: bool,
+    /// The token runs of a near-miss copy that its partners match, within its segments;
+    /// `None` for an exact or gapped copy, whose segments are matched throughout.
+    matched_runs: Option<Vec<(usize, usize)>>,
+    /// The exact fragments coalesced into a near-miss copy: each line they reach is duplicated,
+    /// whatever else of the copy shares the line.
+    exact_runs: Vec<(usize, usize)>,
     /// Sum of segment token counts (the gap tokens are not matched content).
     token_count: usize,
     start_token_index: usize,
@@ -1588,6 +1597,16 @@ struct CountedOccurrence {
     end_line: usize,
 }
 
+impl CountedOccurrence {
+    /// Whether a matched segment overlaps the token range: the gap of a gapped clone is not
+    /// matched content.
+    fn matches_within(&self, (start, end): (usize, usize)) -> bool {
+        self.segments
+            .iter()
+            .any(|&(segment_start, segment_end)| segment_start < end && start < segment_end)
+    }
+}
+
 fn to_counted_groups(
     counted: &IndexMap<std::rc::Rc<str>, Vec<DuplicateCandidate>>,
 ) -> Vec<Vec<CountedOccurrence>> {
@@ -1597,6 +1616,8 @@ fn to_counted_groups(
             .iter()
             .map(|candidate| CountedOccurrence {
                 shared_with_merged_group: false,
+                matched_runs: None,
+                exact_runs: Vec::new(),
                 segments: vec![(candidate.start_token_index, candidate.end_token_index)],
                 token_count: candidate.token_count,
                 start_token_index: candidate.start_token_index,
@@ -1785,6 +1806,8 @@ fn merge_groups(
             CountedOccurrence {
                 // A merged occurrence is a fresh span combination; it inherits no shared marks.
                 shared_with_merged_group: false,
+                matched_runs: None,
+                exact_runs: Vec::new(),
                 segments: [leading.segments.clone(), trailing.segments.clone()].concat(),
                 token_count: leading.token_count + trailing.token_count,
                 start_token_index: leading.start_token_index,
@@ -1802,15 +1825,32 @@ fn merge_groups(
     })
 }
 
-/// A verified near-miss pair as (block, core, block, core); a `None` core is a whole-block match.
-type MatchEdge = (usize, Option<(usize, usize)>, usize, Option<(usize, usize)>);
+/// A verified near-miss pair of blocks; without `cores`, the blocks match as wholes.
+struct MatchEdge {
+    left: usize,
+    right: usize,
+    cores: Option<CorePair>,
+    /// The token runs the pair matches, on the left and on the right.
+    matched_runs: (TokenRuns, TokenRuns),
+}
+
+impl MatchEdge {
+    /// The (block, core) on each side.
+    fn sides(&self) -> [(usize, Option<(usize, usize)>); 2] {
+        let (left_core, right_core) = self.cores.unzip();
+        [(self.left, left_core), (self.right, right_core)]
+    }
+}
 
 /// Detects near-miss (Type-3) clone groups among block candidates the exact pipeline left
 /// unreported: NIL-style n-gram filtration, then pair verification (near_miss::Matcher), then
 /// transitive clustering of verified pairs (crossFileNearMiss.ts applies the same model across
 /// files). A block that matched only locally is reported as its matched cores (overlapping cores
 /// merged), each clustered with its own partners, so code no verified pair matched never counts
-/// as duplicated.
+/// as duplicated. A node overlapping a reported occurrence is an anchor: it joins its group at its
+/// own size, so the copies of a block are listed at the size they matched it, and is marked as
+/// counted elsewhere unless the group takes over the reported occurrences it overlaps. An anchor
+/// still counts the lines its partners match outside those occurrences.
 fn collect_near_miss_groups(
     source: &TokenizedSource<'_>,
     settings: &DuplicationSettings,
@@ -1825,22 +1865,23 @@ fn collect_near_miss_groups(
         return Vec::new();
     }
 
-    // Reported-group indices whose occurrences overlap a token range: near-miss nodes covering
-    // such content anchor comparisons but are never re-reported.
+    // Reported-group indices whose occurrences' matched segments overlap a token range (the gap
+    // of a gapped clone is not reported content): near-miss nodes covering such content anchor
+    // comparisons but are never re-reported.
     let touched_groups_in = |start: usize, end: usize| -> Vec<usize> {
         reported_groups
             .iter()
             .enumerate()
             .filter(|(_, group)| {
-                group.iter().any(|occurrence| {
-                    occurrence.start_token_index < end && start < occurrence.end_token_index
-                })
+                group
+                    .iter()
+                    .any(|occurrence| occurrence.matches_within((start, end)))
             })
             .map(|(group_index, _)| group_index)
             .collect()
     };
 
-    let (symbols, is_content) = to_symbol_stream(tokens);
+    let (symbols, is_content, token_keys) = to_symbol_stream(tokens);
     let statements = top_level_statement_finder(&source.container_statement_ranges);
     let mut blocks: Vec<Block> = comparable
         .iter()
@@ -1848,6 +1889,7 @@ fn collect_near_miss_groups(
             Block::new(
                 &symbols,
                 &is_content,
+                &token_keys,
                 range.start_token_index,
                 range.end_token_index,
                 statements(range.start_token_index, range.end_token_index),
@@ -1860,27 +1902,69 @@ fn collect_near_miss_groups(
         settings.min_similarity_percent,
     );
 
-    let block_touched: Vec<bool> = comparable
-        .iter()
-        .map(|range| !touched_groups_in(range.start_token_index, range.end_token_index).is_empty())
-        .collect();
+    // The lines of a near-miss copy that are duplicated are those a verified partner matches,
+    // which takes the pairing itself. Every pair kept is aligned: verifying it already cost a
+    // longest common subsequence of the same sequences, and any choice among a copy's partners
+    // would let their number or their order decide what the copy counts.
+    //
+    // A pair matching nothing outside reported content has nothing to add to it and is dropped
+    // before clustering, so that it neither collapses the blocks' core nodes nor widens a node
+    // beyond what the pairs clustering it matched. Any other pair is kept however much of it is
+    // reported: what the exact pipeline reports of a copy depends on where its copies stand, and
+    // the lines the copy counts must not.
+    let mut reported_prefix = vec![0usize; tokens.len() + 1];
+    {
+        let mut reported = vec![false; tokens.len()];
+        for occurrence in reported_groups.iter().flatten() {
+            for &(start, end) in &occurrence.segments {
+                reported[start..end].fill(true);
+            }
+        }
+        for (index, &flag) in reported.iter().enumerate() {
+            reported_prefix[index + 1] = reported_prefix[index] + usize::from(flag);
+        }
+    }
+    let reported_count =
+        |(start, end): (usize, usize)| reported_prefix[end] - reported_prefix[start];
+    let unreported_count = |runs: &[(usize, usize)]| -> usize {
+        runs.iter()
+            .map(|&run| run.1 - run.0 - reported_count(run))
+            .sum()
+    };
+    let block_span = |index: usize| {
+        (
+            comparable[index].start_token_index,
+            comparable[index].end_token_index,
+        )
+    };
     let mut edges: Vec<MatchEdge> = Vec::new();
+    let mut add_edge = |left: usize, right: usize, cores: Option<CorePair>, alignment| {
+        let (left_span, right_span) = cores.unwrap_or((block_span(left), block_span(right)));
+        // Fully reported spans need no alignment to be dropped.
+        if unreported_count(&[left_span, right_span]) == 0 {
+            return;
+        }
+        let matched_runs =
+            matcher.align(&blocks[left], &blocks[right], cores, alignment, &token_keys);
+        if unreported_count(&matched_runs.0) + unreported_count(&matched_runs.1) == 0 {
+            return;
+        }
+        edges.push(MatchEdge {
+            left,
+            right,
+            cores,
+            matched_runs,
+        });
+    };
     for_each_candidate_pair(
         &blocks,
         settings.min_similarity_percent,
         |left_index, right_index| match matcher.verify(&blocks[left_index], &blocks[right_index]) {
             None => {}
-            // A whole match between two blocks that both overlap reported content could never join
-            // a group, and recording it would collapse the blocks' core nodes.
-            Some(PairMatch::Whole)
-                if !(block_touched[left_index] && block_touched[right_index]) =>
-            {
-                edges.push((left_index, None, right_index, None))
-            }
-            Some(PairMatch::Whole) => {}
+            Some(PairMatch::Whole(alignment)) => add_edge(left_index, right_index, None, alignment),
             Some(PairMatch::Local(cores)) => {
-                for (left_core, right_core) in cores {
-                    edges.push((left_index, Some(left_core), right_index, Some(right_core)));
+                for (core_pair, alignment) in cores {
+                    add_edge(left_index, right_index, Some(core_pair), alignment);
                 }
             }
         },
@@ -1891,8 +1975,8 @@ fn collect_near_miss_groups(
     // cores matched with different partners fall into separate groups.
     let mut matched_whole = vec![false; comparable.len()];
     let mut local_cores: Vec<Vec<(usize, usize)>> = vec![Vec::new(); comparable.len()];
-    for &(left_index, left_core, right_index, right_core) in &edges {
-        for (index, core) in [(left_index, left_core), (right_index, right_core)] {
+    for edge in &edges {
+        for (index, core) in edge.sides() {
             match core {
                 Some(core) => local_cores[index].push(core),
                 None => matched_whole[index] = true,
@@ -1958,22 +2042,22 @@ fn collect_near_miss_groups(
             touched_groups_in(start, end)
         })
         .collect();
-    for &(left_index, left_core, right_index, right_core) in &edges {
-        let (left_node, right_node) = (
-            node_of(left_index, left_core),
-            node_of(right_index, right_core),
-        );
-        // Two already-reported nodes have nothing new to contribute to each other.
-        if !touched_groups_by_node[left_node].is_empty()
-            && !touched_groups_by_node[right_node].is_empty()
-        {
-            continue;
-        }
+    let edge_nodes: Vec<[usize; 2]> = edges
+        .iter()
+        .map(|edge| edge.sides().map(|(index, core)| node_of(index, core)))
+        .collect();
+    for &[left_node, right_node] in &edge_nodes {
         let left_root = find(&mut parent, left_node);
         let right_root = find(&mut parent, right_node);
         parent[left_root.max(right_root)] = left_root.min(right_root);
     }
 
+    let mut matched_runs_by_node: Vec<Vec<(usize, usize)>> = vec![Vec::new(); node_blocks.len()];
+    for (edge, nodes) in edges.iter().zip(&edge_nodes) {
+        matched_runs_by_node[nodes[0]].extend(&edge.matched_runs.0);
+        matched_runs_by_node[nodes[1]].extend(&edge.matched_runs.1);
+    }
+
     let mut members_by_root: IndexMap<usize, Vec<usize>> = IndexMap::new();
     for node in 0..node_blocks.len() {
         let root = find(&mut parent, node);
@@ -2001,8 +2085,15 @@ fn collect_near_miss_groups(
                 segments.sort_unstable();
                 let whole = spans.iter().any(Option::is_none);
                 let (start, end) = (segments[0].0, segments[segments.len() - 1].1);
+                let matched_runs: Vec<(usize, usize)> = nodes
+                    .iter()
+                    .filter(|&&node| node_blocks[node] == block)
+                    .flat_map(|&node| matched_runs_by_node[node].iter().copied())
+                    .collect();
                 CountedOccurrence {
                     shared_with_merged_group: false,
+                    matched_runs: Some(merge_overlapping_cores(&matched_runs)),
+                    exact_runs: Vec::new(),
                     token_count: segments.iter().map(|segment| segment.1 - segment.0).sum(),
                     segments,
                     start_token_index: start,
@@ -2022,6 +2113,13 @@ fn collect_near_miss_groups(
             .collect()
     };
     let touched_groups_of = |node: usize| &touched_groups_by_node[node];
+    let to_anchor_occurrences = |nodes: &[usize]| -> Vec<CountedOccurrence> {
+        let mut anchors = to_occurrences(nodes);
+        for anchor in &mut anchors {
+            anchor.shared_with_merged_group = true;
+        }
+        anchors
+    };
     let mut groups: Vec<Vec<CountedOccurrence>> = Vec::new();
     for members in members_by_root.values() {
         if members.len() < 2 {
@@ -2044,17 +2142,42 @@ fn collect_near_miss_groups(
             }
             continue;
         }
-        if uncovered.is_empty() {
+        if members.iter().all(|&node| {
+            let (start, end) = node_range(node);
+            reported_count((start, end)) == end - start
+        }) {
             continue;
         }
-        // An anchored cluster extends a reported group only when every occurrence of that group
-        // overlaps one of the cluster's member nodes: an occurrence disjoint from all members
-        // reports content the cluster does not share.
-        let overlaps_member = |occurrence: &CountedOccurrence| {
-            members.iter().any(|&index| {
-                let (start, end) = node_range(index);
-                occurrence.start_token_index < end && start < occurrence.end_token_index
-            })
+        // An anchored cluster extends a reported group only when it stands for the same copies:
+        // every occurrence of the group overlaps the member nodes of exactly one block (one
+        // disjoint from all members reports content the cluster does not share, and one reaching
+        // into two members would make one copy take over lines of the next), exceeds the span of
+        // those nodes by less than `min_tokens` (a copy is listed at the size it matched, not at
+        // that of a clone larger by a reportable part, which may run into code no member
+        // matched), and shares that block with no other occurrence of the group (those repeat
+        // within the copy, not between the copies).
+        let member_block_of = |occurrence: &CountedOccurrence| {
+            let mut blocks = members
+                .iter()
+                .filter(|&&index| occurrence.matches_within(node_range(index)))
+                .map(|&index| node_blocks[index]);
+            let block = blocks.next()?;
+            let mut spans = members
+                .iter()
+                .filter(|&&index| node_blocks[index] == block)
+                .map(|&index| node_range(index));
+            let first = spans.next()?;
+            let span = spans.fold(first, |span, node| (span.0.min(node.0), span.1.max(node.1)));
+            let excess = span.0.saturating_sub(occurrence.start_token_index)
+                + occurrence.end_token_index.saturating_sub(span.1);
+            (blocks.all(|other| other == block) && excess < settings.min_tokens).then_some(block)
+        };
+        let stands_for_members = |group: &[CountedOccurrence]| {
+            let mut blocks: FxHashSet<usize> = FxHashSet::default();
+            !group.is_empty()
+                && group.iter().all(|occurrence| {
+                    member_block_of(occurrence).is_some_and(|block| blocks.insert(block))
+                })
         };
         // Ascending by construction: BTreeSet iteration is sorted and filter preserves order.
         let fully_clustered: Vec<usize> = covered
@@ -2062,18 +2185,15 @@ fn collect_near_miss_groups(
             .flat_map(|&index| touched_groups_of(index).iter().copied())
             .collect::<std::collections::BTreeSet<usize>>()
             .into_iter()
-            .filter(|&group_index| {
-                let group = &reported_groups[group_index];
-                !group.is_empty() && group.iter().all(&overlaps_member)
-            })
+            .filter(|&group_index| stands_for_members(&reported_groups[group_index]))
             .collect();
         if let Some((&target_index, source_indexes)) = fully_clustered.split_first() {
             // Rebuild the component as ONE group with one coalesced occurrence per member block:
-            // the fragments every node of a block overlaps are collected together, since a block's
+            // the fragments the nodes of a block overlap are collected together, since a block's
             // cores are parts of one copy.
-            let mut consumed: FxHashSet<(usize, usize)> = FxHashSet::default();
             let mut merged: Vec<CountedOccurrence> = Vec::new();
             let mut unanchored_nodes: Vec<usize> = Vec::new();
+            let mut anchor_nodes: Vec<usize> = Vec::new();
             let mut nodes_by_block: IndexMap<usize, Vec<usize>> = IndexMap::new();
             for &member_index in members {
                 nodes_by_block
@@ -2082,73 +2202,51 @@ fn collect_near_miss_groups(
                     .push(member_index);
             }
             for block_nodes in nodes_by_block.values() {
-                // Occurrences of ONE group are distinct copies; only fragments from DIFFERENT
-                // groups belong to the same copy. Consecutive position-order slices keep the
-                // coalesced spans disjoint.
-                let mut fragments: Vec<(CountedOccurrence, usize)> = Vec::new();
-                let mut plain_nodes: Vec<usize> = Vec::new();
-                for &node in block_nodes {
-                    let (range_start, range_end) = node_range(node);
-                    let fragment_count = fragments.len();
-                    for &group_index in &fully_clustered {
-                        for (occurrence_index, occurrence) in
-                            reported_groups[group_index].iter().enumerate()
-                        {
-                            if !consumed.contains(&(group_index, occurrence_index))
-                                && occurrence.start_token_index < range_end
-                                && range_start < occurrence.end_token_index
-                            {
-                                consumed.insert((group_index, occurrence_index));
-                                fragments.push((occurrence.clone(), group_index));
-                            }
-                        }
-                    }
-                    if fragments.len() == fragment_count && touched_groups_of(node).is_empty() {
-                        plain_nodes.push(node);
-                    }
-                }
+                let overlaps_block_node = |occurrence: &&CountedOccurrence| {
+                    block_nodes
+                        .iter()
+                        .any(|&node| occurrence.matches_within(node_range(node)))
+                };
+                let mut fragments: Vec<CountedOccurrence> = fully_clustered
+                    .iter()
+                    .flat_map(|&group_index| &reported_groups[group_index])
+                    .filter(overlaps_block_node)
+                    .cloned()
+                    .collect();
                 if fragments.is_empty() {
-                    unanchored_nodes.extend(plain_nodes);
-                } else {
-                    // An untouched core of a block that also holds fragments is part of the same
-                    // copy; a group index no reported group uses keeps it in that copy.
-                    for occurrence in to_occurrences(&plain_nodes) {
-                        fragments.push((occurrence, usize::MAX));
-                    }
-                }
-                fragments.sort_by_key(|(occurrence, _)| {
-                    (occurrence.start_token_index, occurrence.end_token_index)
-                });
-                let mut copy_parts: Vec<CountedOccurrence> = Vec::new();
-                let mut copy_groups: FxHashSet<usize> = FxHashSet::default();
-                for (occurrence, group_index) in fragments {
-                    if copy_groups.contains(&group_index) {
-                        merged.push(coalesce_occurrences(std::mem::take(&mut copy_parts)));
-                        copy_groups.clear();
-                    }
-                    copy_parts.push(occurrence);
-                    copy_groups.insert(group_index);
-                }
-                if !copy_parts.is_empty() {
-                    merged.push(coalesce_occurrences(copy_parts));
+                    let (anchors, plain): (Vec<usize>, Vec<usize>) = block_nodes
+                        .iter()
+                        .partition(|&&node| !touched_groups_of(node).is_empty());
+                    anchor_nodes.extend(anchors);
+                    unanchored_nodes.extend(plain);
+                    continue;
                 }
+                // The block's nodes are part of the same copy as its fragments: a block matched
+                // beyond the fragments it holds is a copy at that size.
+                fragments.extend(to_occurrences(block_nodes));
+                merged.push(coalesce_occurrences(fragments));
             }
             merged.extend(to_occurrences(&unanchored_nodes));
-            merged.sort_by_key(|occurrence| {
-                (occurrence.start_token_index, occurrence.end_token_index)
-            });
             // The rebuild consumed every fully-clustered group, so shared-span marks from earlier
             // partial merges no longer point at a separate merged group.
             for occurrence in &mut merged {
                 occurrence.shared_with_merged_group = false;
             }
+            merged.extend(to_anchor_occurrences(&anchor_nodes));
+            merged.sort_by_key(|occurrence| {
+                (occurrence.start_token_index, occurrence.end_token_index)
+            });
             reported_groups[target_index] = merged;
             for &source_index in source_indexes {
                 reported_groups[source_index].clear();
             }
         } else {
-            let occurrences = to_occurrences(&uncovered);
+            let mut occurrences = to_occurrences(&uncovered);
+            occurrences.extend(to_anchor_occurrences(&covered));
             if occurrences.len() >= 2 {
+                occurrences.sort_by_key(|occurrence| {
+                    (occurrence.start_token_index, occurrence.end_token_index)
+                });
                 groups.push(occurrences);
             }
         }
@@ -2263,8 +2361,41 @@ fn coalesce_occurrences(occurrences: Vec<CountedOccurrence>) -> CountedOccurrenc
             _ => segments.push(segment),
         }
     }
+    // An exact fragment is matched throughout, a near-miss part where its partners match it.
+    let matched_runs = occurrences
+        .iter()
+        .any(|occurrence| occurrence.matched_runs.is_some())
+        .then(|| {
+            let runs: Vec<(usize, usize)> = occurrences
+                .iter()
+                .flat_map(|occurrence| {
+                    occurrence
+                        .matched_runs
+                        .as_ref()
+                        .unwrap_or(&occurrence.segments)
+                        .iter()
+                        .copied()
+                })
+                .collect();
+            merge_overlapping_cores(&runs)
+        });
+    let exact_runs: Vec<(usize, usize)> = occurrences
+        .iter()
+        .flat_map(|occurrence| match occurrence.matched_runs {
+            Some(_) => &occurrence.exact_runs,
+            None => &occurrence.segments,
+        })
+        .copied()
+        .collect();
     CountedOccurrence {
         shared_with_merged_group: false,
+        // Without a near-miss part, the segments are matched throughout and say it all.
+        exact_runs: if matched_runs.is_some() {
+            merge_overlapping_cores(&exact_runs)
+        } else {
+            Vec::new()
+        },
+        matched_runs,
         token_count: segments.iter().map(|segment| segment.1 - segment.0).sum(),
         start_token_index: occurrences
             .iter()
@@ -2300,32 +2431,45 @@ fn merge_overlapping_cores(cores: &[(usize, usize)]) -> Vec<(usize, usize)> {
 /// other token interned from its hash pairs (literal VALUES folded in, unlike the exact
 /// fingerprint's kind tags), plus which tokens are content-bearing (names and literal values).
 /// Interned per call so a file's symbol ids (and thus its n-gram hashes) never depend on which
-/// other files the process measured before it.
-fn to_symbol_stream(tokens: &[Token<'_>]) -> (Vec<i32>, Vec<bool>) {
+/// other files the process measured before it. A symbol id still depends on where in the file its
+/// token first occurs, so each non-identifier token also gets a key of its content alone.
+fn to_symbol_stream(tokens: &[Token<'_>]) -> (Vec<i32>, Vec<bool>, Vec<i64>) {
     let mut symbol_by_token_hashes: FxHashMap<(i32, i32, i32, i32), i32> = FxHashMap::default();
     let mut id_by_identifier: FxHashMap<&str, i32> = FxHashMap::default();
-    tokens
-        .iter()
-        .map(|token| {
-            if token.is_id {
-                let next_id = id_by_identifier.len() as i32;
-                let id = *id_by_identifier
-                    .entry(token.text.as_ref())
-                    .or_insert(next_id);
-                return (-(id + 1), false);
-            }
-            let next_symbol = symbol_by_token_hashes.len() as i32;
-            let symbol = *symbol_by_token_hashes
-                .entry((
-                    token.text_hash,
-                    token.text_hash2,
-                    token.literal_hash.unwrap_or(0),
-                    token.literal_hash2.unwrap_or(0),
-                ))
-                .or_insert(next_symbol);
-            (symbol, token.is_name || token.literal_hash.is_some())
-        })
-        .unzip()
+    let mut symbols = Vec::with_capacity(tokens.len());
+    let mut is_content = Vec::with_capacity(tokens.len());
+    let mut token_keys = Vec::with_capacity(tokens.len());
+    for token in tokens {
+        if token.is_id {
+            let next_id = id_by_identifier.len() as i32;
+            let id = *id_by_identifier
+                .entry(token.text.as_ref())
+                .or_insert(next_id);
+            symbols.push(-(id + 1));
+            is_content.push(false);
+            token_keys.push(0);
+            continue;
+        }
+        let hashes = (
+            token.text_hash,
+            token.text_hash2,
+            token.literal_hash.unwrap_or(0),
+            token.literal_hash2.unwrap_or(0),
+        );
+        let next_symbol = symbol_by_token_hashes.len() as i32;
+        symbols.push(*symbol_by_token_hashes.entry(hashes).or_insert(next_symbol));
+        is_content.push(token.is_name || token.literal_hash.is_some());
+        token_keys.push(token_content_key(hashes));
+    }
+    (symbols, is_content, token_keys)
+}
+
+/// A 53-bit key of a token's two text hashes, each mixed with the matching literal value hash; the
+/// same key as `tokenKey` in crossFileNearMiss.ts, so that both detectors order content alike.
+fn token_content_key((text, text2, literal, literal2): (i32, i32, i32, i32)) -> i64 {
+    let primary = (text ^ literal.wrapping_mul(0x9E37_79B1_u32 as i32)) as u32;
+    let secondary = (text2 ^ literal2.wrapping_mul(0x85EB_CA6B_u32 as i32)) as u32;
+    i64::from(primary) * 0x20_0000 + i64::from(secondary >> 11)
 }
 
 /// Returns a lookup of the outermost container statements inside a token range, excluding a
@@ -2429,23 +2573,44 @@ fn count_redundant_fragments(group: &[CountedOccurrence]) -> usize {
 }
 
 /// Only CODE lines carrying matched tokens count; the unmatched gap of a merged clone stays out
-/// of line coverage.
+/// of line coverage. A line of a near-miss copy counts when its partners match more than half of
+/// its tokens: a line the copy adds or rewrites is not duplicated, while a longest common
+/// subsequence still pairs a stray token of it (a bracket, a separator) with one of the partner.
+/// A line an exact fragment of the copy reaches counts in any case.
 fn collect_matched_lines(
     occurrence: &CountedOccurrence,
     code_line_numbers: &FxHashSet<usize>,
     tokens: &[Token<'_>],
 ) -> FxHashSet<usize> {
-    let mut lines: FxHashSet<usize> = FxHashSet::default();
+    let mut token_counts_by_row: FxHashMap<usize, (usize, usize)> = FxHashMap::default();
+    let mut runs = occurrence.matched_runs.iter().flatten().peekable();
     for &(segment_start, segment_end) in &occurrence.segments {
-        for token in &tokens[segment_start..segment_end.min(tokens.len())] {
+        let segment = tokens.iter().enumerate().take(segment_end);
+        for (index, token) in segment.skip(segment_start) {
+            while runs.next_if(|run| run.1 <= index).is_some() {}
+            let matched =
+                occurrence.matched_runs.is_none() || runs.peek().is_some_and(|run| run.0 <= index);
             for row in token.start_row..=token.end_row {
-                if code_line_numbers.contains(&(row + 1)) {
-                    lines.insert(row + 1);
-                }
+                let (matched_count, token_count) = token_counts_by_row.entry(row + 1).or_default();
+                *matched_count += usize::from(matched);
+                *token_count += 1;
             }
         }
     }
-    lines
+    let exact_rows: FxHashSet<usize> = occurrence
+        .exact_runs
+        .iter()
+        .flat_map(|&(start, end)| &tokens[start..end])
+        .flat_map(|token| token.start_row + 1..=token.end_row + 1)
+        .collect();
+    token_counts_by_row
+        .into_iter()
+        .filter(|(line, (matched_count, token_count))| {
+            code_line_numbers.contains(line)
+                && (matched_count * 2 > *token_count || exact_rows.contains(line))
+        })
+        .map(|(line, _)| line)
+        .collect()
 }
 
 fn summarize_duplicates(