code-gauge 4.3.1 → 4.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -4
- package/dist/crossFileDuplication.cjs +1 -1
- package/dist/crossFileDuplication.cjs.map +1 -1
- package/dist/crossFileDuplication.d.ts +11 -4
- package/dist/crossFileDuplication.js +1 -1
- package/dist/crossFileDuplication.js.map +1 -1
- package/dist/crossFileNearMiss.cjs +2 -0
- package/dist/crossFileNearMiss.cjs.map +1 -0
- package/dist/crossFileNearMiss.d.ts +27 -0
- package/dist/crossFileNearMiss.js +2 -0
- package/dist/crossFileNearMiss.js.map +1 -0
- package/dist/diffCommand.cjs +1 -1
- package/dist/diffCommand.cjs.map +1 -1
- package/dist/diffCommand.js +3 -3
- package/dist/diffCommand.js.map +1 -1
- package/dist/duplication.cjs +1 -1
- package/dist/duplication.cjs.map +1 -1
- package/dist/duplication.d.ts +11 -0
- package/dist/duplication.js +1 -1
- package/dist/duplication.js.map +1 -1
- package/dist/metrics.cjs +1 -1
- package/dist/metrics.cjs.map +1 -1
- package/dist/metrics.d.ts +10 -0
- package/dist/metrics.js +1 -1
- package/dist/metrics.js.map +1 -1
- package/dist/nativeMetrics.cjs +2 -2
- package/dist/nativeMetrics.cjs.map +1 -1
- package/dist/nativeMetrics.d.ts +7 -2
- package/dist/nativeMetrics.js +2 -2
- package/dist/nativeMetrics.js.map +1 -1
- package/dist/scan.cjs +1 -1
- package/dist/scan.cjs.map +1 -1
- package/dist/scan.d.ts +12 -1
- package/dist/scan.js +1 -1
- package/dist/scan.js.map +1 -1
- package/dist/types.d.ts +11 -1
- package/native/src/complexity.rs +23 -7
- package/native/src/dep_degree.rs +2 -3
- package/native/src/duplication.rs +170 -115
- package/native/src/functions.rs +1 -1
- package/native/src/languages.rs +17 -0
- package/native/src/lib.rs +6 -2
- package/native/src/measure.rs +118 -13
- package/native/src/types.rs +6 -0
- package/package.json +8 -8
package/native/src/complexity.rs
CHANGED
|
@@ -136,6 +136,16 @@ struct FunctionBodyPass<'sets, 'code, 'source> {
|
|
|
136
136
|
code: &'code Source<'source>,
|
|
137
137
|
frames: Vec<FunctionBodyFrame>,
|
|
138
138
|
results: HashMap<usize, FunctionBodyMetrics>,
|
|
139
|
+
/// Cyclomatic decisions inside class bodies nested in functions, which no function owns.
|
|
140
|
+
nested_class_decisions: u64,
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/// Per-function body metrics, plus the cyclomatic decisions no function body owns.
|
|
144
|
+
pub struct BodyMetrics {
|
|
145
|
+
pub by_function: HashMap<usize, FunctionBodyMetrics>,
|
|
146
|
+
/// Cyclomatic decisions outside every function body (top-level statements, field initializers,
|
|
147
|
+
/// including those of classes nested in functions).
|
|
148
|
+
pub top_level_decisions: u64,
|
|
139
149
|
}
|
|
140
150
|
|
|
141
151
|
/// Per-function complexity and NCSS for every function boundary, in one post-order pass so each
|
|
@@ -150,16 +160,20 @@ pub fn measure_function_body_metrics(
|
|
|
150
160
|
root: Node<'_>,
|
|
151
161
|
sets: &LanguageSets,
|
|
152
162
|
code: &Source<'_>,
|
|
153
|
-
) ->
|
|
163
|
+
) -> BodyMetrics {
|
|
154
164
|
let mut pass = FunctionBodyPass {
|
|
155
165
|
sets,
|
|
156
166
|
code,
|
|
157
|
-
// frames[0] is a sentinel for top-level code; its
|
|
167
|
+
// frames[0] is a sentinel for top-level code; only its cyclomatic decisions are kept.
|
|
158
168
|
frames: vec![FunctionBodyFrame::new(0, 0)],
|
|
159
169
|
results: HashMap::new(),
|
|
170
|
+
nested_class_decisions: 0,
|
|
160
171
|
};
|
|
161
172
|
pass.visit(root, 0, 0, false, false, false);
|
|
162
|
-
|
|
173
|
+
BodyMetrics {
|
|
174
|
+
by_function: pass.results,
|
|
175
|
+
top_level_decisions: pass.frames[0].cyclomatic_complexity - 1 + pass.nested_class_decisions,
|
|
176
|
+
}
|
|
163
177
|
}
|
|
164
178
|
|
|
165
179
|
impl FunctionBodyPass<'_, '_, '_> {
|
|
@@ -216,10 +230,12 @@ impl FunctionBodyPass<'_, '_, '_> {
|
|
|
216
230
|
|
|
217
231
|
// Each branch, short-circuit operator, and pattern guard adds one path (McCabe; NIST SP
|
|
218
232
|
// 500-235 §4); `else` adds none.
|
|
219
|
-
if
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
233
|
+
if is_decision || is_boolean_operator(current, self.code) || is_pattern_guard(current) {
|
|
234
|
+
if counts_for_own_body {
|
|
235
|
+
self.top_frame().cyclomatic_complexity += 1;
|
|
236
|
+
} else if !opens_frame {
|
|
237
|
+
self.nested_class_decisions += 1;
|
|
238
|
+
}
|
|
223
239
|
}
|
|
224
240
|
if is_decision && !is_case_clause {
|
|
225
241
|
if is_continuation {
|
package/native/src/dep_degree.rs
CHANGED
|
@@ -26,7 +26,6 @@ const COMPOUND_ASSIGNMENT_OPERATORS: &[&str] = &[
|
|
|
26
26
|
|
|
27
27
|
/// Parent type -> field under which an identifier is a definition target even when a type
|
|
28
28
|
/// annotation separates it from the `=` token, plus loop bindings with no assignment token at all.
|
|
29
|
-
/// Mirrors definitionFieldByParentType in metrics.ts.
|
|
30
29
|
const DEFINITION_FIELD_BY_PARENT_TYPE: &[(&str, &str)] = &[
|
|
31
30
|
("variable_declarator", "name"),
|
|
32
31
|
("let_declaration", "pattern"),
|
|
@@ -109,7 +108,7 @@ struct DepDegreeLeaf<'t> {
|
|
|
109
108
|
scope: String,
|
|
110
109
|
}
|
|
111
110
|
|
|
112
|
-
/// Approximate def-use pairs of the function's subtree
|
|
111
|
+
/// Approximate def-use pairs of the function's subtree.
|
|
113
112
|
pub fn measure_dep_degree(
|
|
114
113
|
function_node: Node<'_>,
|
|
115
114
|
code: &Source<'_>,
|
|
@@ -362,7 +361,7 @@ fn unwrap_declarator_wrappers<'t>(leaf: &DepDegreeLeaf<'t>) -> (Node<'t>, Option
|
|
|
362
361
|
(current, field_name)
|
|
363
362
|
}
|
|
364
363
|
|
|
365
|
-
///
|
|
364
|
+
/// Whether an identifier defines a parameter: an ancestor reached through declarator wrappers
|
|
366
365
|
/// (C/C++ function-pointer or array parameters) is a parameter-ish node, or the identifier
|
|
367
366
|
/// directly occupies a parameter field; type annotations and default values bind nothing.
|
|
368
367
|
fn is_parameter_definition(leaf: &DepDegreeLeaf<'_>) -> bool {
|
|
@@ -10,7 +10,7 @@ use crate::types::{
|
|
|
10
10
|
};
|
|
11
11
|
use crate::util::{all_children, is_identifier_leaf, named_children, node_text, to_int32, Source};
|
|
12
12
|
|
|
13
|
-
/// Block-like nodes considered as whole-subtree duplicate candidates
|
|
13
|
+
/// Block-like nodes considered as whole-subtree duplicate candidates.
|
|
14
14
|
const DUPLICATE_BLOCK_TYPES: &[&str] = &[
|
|
15
15
|
"statement_block",
|
|
16
16
|
"block",
|
|
@@ -314,11 +314,12 @@ impl Default for DuplicationSettings {
|
|
|
314
314
|
}
|
|
315
315
|
}
|
|
316
316
|
}
|
|
317
|
-
/// N-gram size for the near-miss candidate index (NIL's default);
|
|
317
|
+
/// N-gram size for the near-miss candidate index (NIL's default); shared with crossFileNearMiss.ts.
|
|
318
318
|
const NEAR_MISS_NGRAM_SIZE: usize = 5;
|
|
319
|
-
/// Filtration threshold: shared distinct n-grams over the smaller set;
|
|
319
|
+
/// Filtration threshold: shared distinct n-grams over the smaller set; shared with crossFileNearMiss.ts.
|
|
320
320
|
const NEAR_MISS_FILTRATION_PERCENT: usize = 10;
|
|
321
|
-
/// Exclusive bound on shared content-bearing tokens (names and literal values);
|
|
321
|
+
/// Exclusive bound on shared content-bearing tokens (names and literal values); shared with
|
|
322
|
+
/// crossFileNearMiss.ts.
|
|
322
323
|
const MIN_CONTENT_SIMILARITY_PERCENT: usize = 50;
|
|
323
324
|
|
|
324
325
|
/// See isLiteralDense in duplication.ts: >= 20% literal values marks a region as data-like.
|
|
@@ -377,15 +378,17 @@ struct DuplicateCandidate {
|
|
|
377
378
|
end_line: usize,
|
|
378
379
|
}
|
|
379
380
|
|
|
380
|
-
///
|
|
381
|
-
///
|
|
382
|
-
pub
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
381
|
+
/// A file's normalized token stream with the block and statement structure clone detection
|
|
382
|
+
/// matches over, built once per parse and shared by within-file and cross-file detection.
|
|
383
|
+
pub struct TokenizedSource<'a> {
|
|
384
|
+
tokens: Vec<Token<'a>>,
|
|
385
|
+
block_ranges: Vec<TokenRange>,
|
|
386
|
+
container_statement_ranges: Vec<Vec<TokenRange>>,
|
|
387
|
+
literal_count_prefix: Vec<usize>,
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
pub fn tokenize<'a>(root: Node<'_>, code: &Source<'a>) -> TokenizedSource<'a> {
|
|
391
|
+
let mut tokens: Vec<Token<'a>> = Vec::new();
|
|
389
392
|
let mut block_ranges: Vec<TokenRange> = Vec::new();
|
|
390
393
|
let mut container_statement_ranges: Vec<Vec<TokenRange>> = Vec::new();
|
|
391
394
|
collect_tokens(
|
|
@@ -395,66 +398,71 @@ pub fn measure_duplication(
|
|
|
395
398
|
&mut block_ranges,
|
|
396
399
|
&mut container_statement_ranges,
|
|
397
400
|
);
|
|
398
|
-
|
|
399
401
|
let literal_count_prefix = build_literal_count_prefix(&tokens);
|
|
402
|
+
TokenizedSource {
|
|
403
|
+
tokens,
|
|
404
|
+
block_ranges,
|
|
405
|
+
container_statement_ranges,
|
|
406
|
+
literal_count_prefix,
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
/// Detects copy-pasted regions within a file. Fingerprints replicate the JavaScript int32 hash
|
|
411
|
+
/// arithmetic of fingerprintKey in duplication.ts, so its candidates group with the window
|
|
412
|
+
/// candidates cross-file matching fingerprints in TypeScript.
|
|
413
|
+
pub fn measure_duplication(
|
|
414
|
+
source: &TokenizedSource<'_>,
|
|
415
|
+
code_line_numbers: &HashSet<usize>,
|
|
416
|
+
settings: &DuplicationSettings,
|
|
417
|
+
) -> DuplicationMetrics {
|
|
418
|
+
let tokens = &source.tokens;
|
|
419
|
+
let literal_count_prefix = &source.literal_count_prefix;
|
|
400
420
|
let mut candidates = collect_block_candidates(
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
&block_ranges,
|
|
421
|
+
tokens,
|
|
422
|
+
literal_count_prefix,
|
|
423
|
+
&source.block_ranges,
|
|
404
424
|
settings.min_tokens,
|
|
405
425
|
);
|
|
406
426
|
candidates.extend(collect_sequence_candidates(
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
&container_statement_ranges,
|
|
427
|
+
tokens,
|
|
428
|
+
literal_count_prefix,
|
|
429
|
+
&source.container_statement_ranges,
|
|
410
430
|
settings.min_tokens,
|
|
411
431
|
));
|
|
412
432
|
let counted = select_maximal_duplicates(candidates);
|
|
413
433
|
let mut groups = merge_adjacent_groups(to_counted_groups(&counted), settings.max_gap_tokens);
|
|
414
|
-
let near_miss = collect_near_miss_groups(
|
|
415
|
-
&tokens,
|
|
416
|
-
&literal_count_prefix,
|
|
417
|
-
&block_ranges,
|
|
418
|
-
settings,
|
|
419
|
-
&mut groups,
|
|
420
|
-
);
|
|
434
|
+
let near_miss = collect_near_miss_groups(source, settings, &mut groups);
|
|
421
435
|
// Near-miss clustering can merge exact groups away, leaving empty entries behind.
|
|
422
436
|
groups.retain(|group| !group.is_empty());
|
|
423
437
|
groups.extend(near_miss);
|
|
424
|
-
summarize_duplicates(&groups, code_line_numbers,
|
|
438
|
+
summarize_duplicates(&groups, code_line_numbers, tokens)
|
|
425
439
|
}
|
|
426
440
|
|
|
427
441
|
/// Collects one file's contribution to cross-file clone detection: catalogued candidates (whole
|
|
428
|
-
/// block subtrees plus each statement container's full run)
|
|
429
|
-
///
|
|
430
|
-
///
|
|
431
|
-
///
|
|
442
|
+
/// block subtrees plus each statement container's full run), the normalized token stream and
|
|
443
|
+
/// statement structure, and the blocks near-miss comparison considers. Source indexes are emitted
|
|
444
|
+
/// in UTF-16 code units (the tree is parsed from UTF-16, so node byte offsets are halved) to match
|
|
445
|
+
/// JavaScript string indexes.
|
|
432
446
|
pub fn collect_cross_file_file_data(
|
|
433
|
-
|
|
434
|
-
code: &Source<'_>,
|
|
447
|
+
source: &TokenizedSource<'_>,
|
|
435
448
|
min_tokens: usize,
|
|
436
449
|
) -> (
|
|
437
450
|
Vec<CrossFileCandidate>,
|
|
438
451
|
Vec<CrossFileToken>,
|
|
439
452
|
Vec<Vec<CrossFileTokenRange>>,
|
|
453
|
+
Vec<CrossFileTokenRange>,
|
|
440
454
|
) {
|
|
441
|
-
let
|
|
442
|
-
let
|
|
443
|
-
let mut
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
&mut block_ranges,
|
|
449
|
-
&mut container_statement_ranges,
|
|
455
|
+
let tokens = &source.tokens;
|
|
456
|
+
let literal_count_prefix = &source.literal_count_prefix;
|
|
457
|
+
let mut candidates = collect_block_candidates(
|
|
458
|
+
tokens,
|
|
459
|
+
literal_count_prefix,
|
|
460
|
+
&source.block_ranges,
|
|
461
|
+
min_tokens,
|
|
450
462
|
);
|
|
451
|
-
let literal_count_prefix = build_literal_count_prefix(&tokens);
|
|
452
|
-
|
|
453
|
-
let mut candidates =
|
|
454
|
-
collect_block_candidates(&tokens, &literal_count_prefix, &block_ranges, min_tokens);
|
|
455
463
|
// Single-statement containers are catalogued too: a file whose only top-level statement is not
|
|
456
464
|
// a block type (a lone exported table) must still be matchable when wholly copied.
|
|
457
|
-
for statements in &container_statement_ranges {
|
|
465
|
+
for statements in &source.container_statement_ranges {
|
|
458
466
|
let (Some(first), Some(last)) = (statements.first(), statements.last()) else {
|
|
459
467
|
continue;
|
|
460
468
|
};
|
|
@@ -465,8 +473,8 @@ pub fn collect_cross_file_file_data(
|
|
|
465
473
|
let fingerprint = format!(
|
|
466
474
|
"s:{}",
|
|
467
475
|
fingerprint_key(
|
|
468
|
-
|
|
469
|
-
|
|
476
|
+
tokens,
|
|
477
|
+
literal_count_prefix,
|
|
470
478
|
first.start_token_index,
|
|
471
479
|
last.end_token_index
|
|
472
480
|
)
|
|
@@ -507,29 +515,34 @@ pub fn collect_cross_file_file_data(
|
|
|
507
515
|
end_row: token.end_row,
|
|
508
516
|
})
|
|
509
517
|
.collect();
|
|
510
|
-
let container_statement_payloads =
|
|
518
|
+
let container_statement_payloads = source
|
|
519
|
+
.container_statement_ranges
|
|
511
520
|
.iter()
|
|
512
|
-
.map(|statements|
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
end_token_index: range.end_token_index,
|
|
518
|
-
start_index: range.start_index / 2,
|
|
519
|
-
end_index: range.end_index / 2,
|
|
520
|
-
start_line: range.start_line,
|
|
521
|
-
end_line: range.end_line,
|
|
522
|
-
})
|
|
523
|
-
.collect()
|
|
524
|
-
})
|
|
521
|
+
.map(|statements| statements.iter().map(to_token_range_payload).collect())
|
|
522
|
+
.collect();
|
|
523
|
+
let near_miss_block_payloads = select_near_miss_blocks(source, min_tokens)
|
|
524
|
+
.into_iter()
|
|
525
|
+
.map(to_token_range_payload)
|
|
525
526
|
.collect();
|
|
526
527
|
(
|
|
527
528
|
candidate_payloads,
|
|
528
529
|
token_payloads,
|
|
529
530
|
container_statement_payloads,
|
|
531
|
+
near_miss_block_payloads,
|
|
530
532
|
)
|
|
531
533
|
}
|
|
532
534
|
|
|
535
|
+
fn to_token_range_payload(range: &TokenRange) -> CrossFileTokenRange {
|
|
536
|
+
CrossFileTokenRange {
|
|
537
|
+
start_token_index: range.start_token_index,
|
|
538
|
+
end_token_index: range.end_token_index,
|
|
539
|
+
start_index: range.start_index / 2,
|
|
540
|
+
end_index: range.end_index / 2,
|
|
541
|
+
start_line: range.start_line,
|
|
542
|
+
end_line: range.end_line,
|
|
543
|
+
}
|
|
544
|
+
}
|
|
545
|
+
|
|
533
546
|
fn collect_tokens<'a>(
|
|
534
547
|
root: Node<'_>,
|
|
535
548
|
code: &Source<'a>,
|
|
@@ -577,8 +590,9 @@ fn collect_tokens<'a>(
|
|
|
577
590
|
statement_ranges.push(child_range);
|
|
578
591
|
}
|
|
579
592
|
}
|
|
580
|
-
// Single-statement containers are recorded too
|
|
581
|
-
//
|
|
593
|
+
// Single-statement containers are recorded too: window enumeration needs two
|
|
594
|
+
// statements and yields nothing for them, but cross-file matching catalogues each
|
|
595
|
+
// container's full run.
|
|
582
596
|
if is_container && !statement_ranges.is_empty() {
|
|
583
597
|
container_statement_ranges.push(statement_ranges);
|
|
584
598
|
}
|
|
@@ -731,8 +745,8 @@ fn make_text_token<'a>(
|
|
|
731
745
|
}
|
|
732
746
|
}
|
|
733
747
|
|
|
734
|
-
/// The value of a literal as folded into literal-dense fingerprints
|
|
735
|
-
///
|
|
748
|
+
/// The value of a literal as folded into literal-dense fingerprints, independent of its delimiter
|
|
749
|
+
/// spelling (quote style, C# verbatim prefix) so equal values in differently quoted copies match.
|
|
736
750
|
fn literal_value_text<'a>(node: Node<'_>, kind: &str, code: &Source<'a>) -> Cow<'a, str> {
|
|
737
751
|
if kind != "#str" && kind != "#char" {
|
|
738
752
|
return Cow::Borrowed(node_text(node, code));
|
|
@@ -759,8 +773,8 @@ fn literal_value_text<'a>(node: Node<'_>, kind: &str, code: &Source<'a>) -> Cow<
|
|
|
759
773
|
Cow::Borrowed(strip_matching_quotes(text))
|
|
760
774
|
}
|
|
761
775
|
|
|
762
|
-
/// Strips one matching pair of surrounding ASCII quotes,
|
|
763
|
-
///
|
|
776
|
+
/// Strips one matching pair of surrounding ASCII quotes (quote characters are ASCII, so byte
|
|
777
|
+
/// indexing is UTF-8 safe).
|
|
764
778
|
fn strip_matching_quotes(text: &str) -> &str {
|
|
765
779
|
let bytes = text.as_bytes();
|
|
766
780
|
if bytes.len() >= 2 {
|
|
@@ -989,7 +1003,7 @@ struct ContainerWindows {
|
|
|
989
1003
|
statement_hashes: Vec<i32>,
|
|
990
1004
|
}
|
|
991
1005
|
|
|
992
|
-
/// Enumerates runs of consecutive sibling statements; see
|
|
1006
|
+
/// Enumerates runs of consecutive sibling statements; see collectSequenceWindowCandidates in
|
|
993
1007
|
/// duplication.ts for the maximality and sub-window rules replicated here.
|
|
994
1008
|
fn collect_sequence_candidates(
|
|
995
1009
|
tokens: &[Token<'_>],
|
|
@@ -1319,7 +1333,7 @@ fn combine_hashes(hash: i64, value: i64) -> i64 {
|
|
|
1319
1333
|
(to_int32(hash).wrapping_mul(31)) as i64 + value
|
|
1320
1334
|
}
|
|
1321
1335
|
|
|
1322
|
-
/// Keeps only maximal, non-overlapping duplicates; see
|
|
1336
|
+
/// Keeps only maximal, non-overlapping duplicates; see selectMaximalGroups in duplicateSelection.ts.
|
|
1323
1337
|
fn select_maximal_duplicates(
|
|
1324
1338
|
candidates: Vec<DuplicateCandidate>,
|
|
1325
1339
|
) -> IndexMap<std::rc::Rc<str>, Vec<DuplicateCandidate>> {
|
|
@@ -1474,8 +1488,9 @@ fn merge_adjacent_groups(
|
|
|
1474
1488
|
let mut restart = true;
|
|
1475
1489
|
while restart {
|
|
1476
1490
|
restart = false;
|
|
1477
|
-
|
|
1478
|
-
|
|
1491
|
+
let partners_by_group = collect_gap_adjacent_partners(&groups, max_gap_tokens);
|
|
1492
|
+
'outer: for (left_index, partners) in partners_by_group.iter().enumerate() {
|
|
1493
|
+
for &right_index in partners {
|
|
1479
1494
|
let forward =
|
|
1480
1495
|
merge_groups(&groups[left_index], &groups[right_index], max_gap_tokens);
|
|
1481
1496
|
let swapped = forward.is_none();
|
|
@@ -1516,6 +1531,43 @@ fn merge_adjacent_groups(
|
|
|
1516
1531
|
groups
|
|
1517
1532
|
}
|
|
1518
1533
|
|
|
1534
|
+
/// Per group index, the ascending indexes of later groups that merge_groups can pair with it;
|
|
1535
|
+
/// see collectGapAdjacentPartners in duplication.ts.
|
|
1536
|
+
fn collect_gap_adjacent_partners(
|
|
1537
|
+
groups: &[Vec<CountedOccurrence>],
|
|
1538
|
+
max_gap_tokens: usize,
|
|
1539
|
+
) -> Vec<Vec<usize>> {
|
|
1540
|
+
let mut starts: Vec<(usize, usize)> = groups
|
|
1541
|
+
.iter()
|
|
1542
|
+
.enumerate()
|
|
1543
|
+
.flat_map(|(group_index, group)| {
|
|
1544
|
+
group
|
|
1545
|
+
.iter()
|
|
1546
|
+
.map(move |occurrence| (occurrence.start_token_index, group_index))
|
|
1547
|
+
})
|
|
1548
|
+
.collect();
|
|
1549
|
+
starts.sort_unstable();
|
|
1550
|
+
let mut partners: Vec<std::collections::BTreeSet<usize>> =
|
|
1551
|
+
vec![std::collections::BTreeSet::new(); groups.len()];
|
|
1552
|
+
for (group_index, group) in groups.iter().enumerate() {
|
|
1553
|
+
for occurrence in group {
|
|
1554
|
+
let first = starts.partition_point(|&(start, _)| start < occurrence.end_token_index);
|
|
1555
|
+
for &(start, other) in &starts[first..] {
|
|
1556
|
+
if start > occurrence.end_token_index + max_gap_tokens {
|
|
1557
|
+
break;
|
|
1558
|
+
}
|
|
1559
|
+
if other != group_index {
|
|
1560
|
+
partners[other.min(group_index)].insert(other.max(group_index));
|
|
1561
|
+
}
|
|
1562
|
+
}
|
|
1563
|
+
}
|
|
1564
|
+
}
|
|
1565
|
+
partners
|
|
1566
|
+
.into_iter()
|
|
1567
|
+
.map(|set| set.into_iter().collect())
|
|
1568
|
+
.collect()
|
|
1569
|
+
}
|
|
1570
|
+
|
|
1519
1571
|
fn group_sort_key(group: &[CountedOccurrence]) -> (usize, usize) {
|
|
1520
1572
|
group
|
|
1521
1573
|
.first()
|
|
@@ -1612,41 +1664,19 @@ fn merge_groups(
|
|
|
1612
1664
|
}
|
|
1613
1665
|
|
|
1614
1666
|
/// Detects near-miss (Type-3) clone groups among block candidates the exact pipeline left
|
|
1615
|
-
/// unreported
|
|
1616
|
-
///
|
|
1617
|
-
///
|
|
1667
|
+
/// unreported: NIL-style n-gram filtration, then token-level LCS with NiCad-style per-fragment
|
|
1668
|
+
/// similarity, then transitive clustering of verified pairs (crossFileNearMiss.ts applies the same
|
|
1669
|
+
/// model across files).
|
|
1618
1670
|
fn collect_near_miss_groups(
|
|
1619
|
-
|
|
1620
|
-
literal_count_prefix: &[usize],
|
|
1621
|
-
block_ranges: &[TokenRange],
|
|
1671
|
+
source: &TokenizedSource<'_>,
|
|
1622
1672
|
settings: &DuplicationSettings,
|
|
1623
1673
|
reported_groups: &mut [Vec<CountedOccurrence>],
|
|
1624
1674
|
) -> Vec<Vec<CountedOccurrence>> {
|
|
1625
1675
|
if settings.min_similarity_percent >= 100 {
|
|
1626
1676
|
return Vec::new();
|
|
1627
1677
|
}
|
|
1628
|
-
let
|
|
1629
|
-
|
|
1630
|
-
.filter(|range| {
|
|
1631
|
-
let token_count = range.end_token_index - range.start_token_index;
|
|
1632
|
-
let literal_count = literal_count_prefix
|
|
1633
|
-
.get(range.end_token_index)
|
|
1634
|
-
.copied()
|
|
1635
|
-
.unwrap_or(0)
|
|
1636
|
-
- literal_count_prefix
|
|
1637
|
-
.get(range.start_token_index)
|
|
1638
|
-
.copied()
|
|
1639
|
-
.unwrap_or(0);
|
|
1640
|
-
token_count >= settings.min_tokens && !is_literal_dense(literal_count, token_count)
|
|
1641
|
-
})
|
|
1642
|
-
.collect();
|
|
1643
|
-
eligible.sort_by_key(|range| {
|
|
1644
|
-
(
|
|
1645
|
-
range.start_token_index,
|
|
1646
|
-
std::cmp::Reverse(range.end_token_index),
|
|
1647
|
-
)
|
|
1648
|
-
});
|
|
1649
|
-
let comparable = select_comparable_blocks(&eligible);
|
|
1678
|
+
let tokens = &source.tokens;
|
|
1679
|
+
let comparable = select_near_miss_blocks(source, settings.min_tokens);
|
|
1650
1680
|
if comparable.len() < 2 {
|
|
1651
1681
|
return Vec::new();
|
|
1652
1682
|
}
|
|
@@ -1712,7 +1742,7 @@ fn collect_near_miss_groups(
|
|
|
1712
1742
|
continue;
|
|
1713
1743
|
}
|
|
1714
1744
|
// A structural match must be backed by shared content (names and literal values); the
|
|
1715
|
-
// bound is exclusive, matching
|
|
1745
|
+
// bound is exclusive, matching crossFileNearMiss.ts.
|
|
1716
1746
|
if content_overlap(left, right) * 100
|
|
1717
1747
|
<= MIN_CONTENT_SIMILARITY_PERCENT * left.content_total.max(right.content_total)
|
|
1718
1748
|
{
|
|
@@ -1769,8 +1799,9 @@ fn collect_near_miss_groups(
|
|
|
1769
1799
|
if uncovered.is_empty() {
|
|
1770
1800
|
continue;
|
|
1771
1801
|
}
|
|
1772
|
-
// An anchored cluster extends a reported group only when
|
|
1773
|
-
// the cluster
|
|
1802
|
+
// An anchored cluster extends a reported group only when every occurrence of that group
|
|
1803
|
+
// overlaps one of the cluster's member blocks: an occurrence disjoint from all members
|
|
1804
|
+
// reports content the cluster does not share.
|
|
1774
1805
|
let overlaps_member = |occurrence: &CountedOccurrence| {
|
|
1775
1806
|
members.iter().any(|&index| {
|
|
1776
1807
|
let range = comparable[index];
|
|
@@ -1790,15 +1821,14 @@ fn collect_near_miss_groups(
|
|
|
1790
1821
|
})
|
|
1791
1822
|
.collect();
|
|
1792
1823
|
if let Some((&target_index, source_indexes)) = fully_clustered.split_first() {
|
|
1793
|
-
// Rebuild the component as ONE group with one coalesced occurrence per member block
|
|
1794
|
-
// see collectNearMissGroups in duplication.ts.
|
|
1824
|
+
// Rebuild the component as ONE group with one coalesced occurrence per member block.
|
|
1795
1825
|
let mut consumed: HashSet<(usize, usize)> = HashSet::new();
|
|
1796
1826
|
let mut merged: Vec<CountedOccurrence> = Vec::new();
|
|
1797
1827
|
for &member_index in members {
|
|
1798
1828
|
let range = comparable[member_index];
|
|
1799
1829
|
// Occurrences of ONE group are distinct copies; only fragments from DIFFERENT
|
|
1800
1830
|
// groups belong to the same copy. Consecutive position-order slices keep the
|
|
1801
|
-
// coalesced spans disjoint
|
|
1831
|
+
// coalesced spans disjoint.
|
|
1802
1832
|
let mut fragments: Vec<(CountedOccurrence, usize)> = Vec::new();
|
|
1803
1833
|
for &group_index in &fully_clustered {
|
|
1804
1834
|
for (occurrence_index, occurrence) in
|
|
@@ -1859,9 +1889,35 @@ fn collect_near_miss_groups(
|
|
|
1859
1889
|
groups
|
|
1860
1890
|
}
|
|
1861
1891
|
|
|
1862
|
-
///
|
|
1863
|
-
///
|
|
1864
|
-
///
|
|
1892
|
+
/// The mutually disjoint blocks near-miss comparison considers: at least `min_tokens` long, not
|
|
1893
|
+
/// literal-dense (data tables are compared by value, not shape), and selected by
|
|
1894
|
+
/// select_comparable_blocks.
|
|
1895
|
+
fn select_near_miss_blocks<'s>(
|
|
1896
|
+
source: &'s TokenizedSource<'_>,
|
|
1897
|
+
min_tokens: usize,
|
|
1898
|
+
) -> Vec<&'s TokenRange> {
|
|
1899
|
+
let literal_count_prefix = &source.literal_count_prefix;
|
|
1900
|
+
let mut eligible: Vec<&TokenRange> = source
|
|
1901
|
+
.block_ranges
|
|
1902
|
+
.iter()
|
|
1903
|
+
.filter(|range| {
|
|
1904
|
+
let token_count = range.end_token_index - range.start_token_index;
|
|
1905
|
+
let literal_count = literal_count_prefix[range.end_token_index]
|
|
1906
|
+
- literal_count_prefix[range.start_token_index];
|
|
1907
|
+
token_count >= min_tokens && !is_literal_dense(literal_count, token_count)
|
|
1908
|
+
})
|
|
1909
|
+
.collect();
|
|
1910
|
+
eligible.sort_by_key(|range| {
|
|
1911
|
+
(
|
|
1912
|
+
range.start_token_index,
|
|
1913
|
+
std::cmp::Reverse(range.end_token_index),
|
|
1914
|
+
)
|
|
1915
|
+
});
|
|
1916
|
+
select_comparable_blocks(&eligible)
|
|
1917
|
+
}
|
|
1918
|
+
|
|
1919
|
+
/// Keeps the block ranges the near-miss phase compares: wrappers whose subtree branches into two
|
|
1920
|
+
/// or more disjoint eligible sub-blocks are descended through; linear chains keep their top.
|
|
1865
1921
|
fn select_comparable_blocks<'a>(eligible: &[&'a TokenRange]) -> Vec<&'a TokenRange> {
|
|
1866
1922
|
struct ForestNode<'a> {
|
|
1867
1923
|
range: &'a TokenRange,
|
|
@@ -1919,8 +1975,7 @@ fn select_comparable_blocks<'a>(eligible: &[&'a TokenRange]) -> Vec<&'a TokenRan
|
|
|
1919
1975
|
kept
|
|
1920
1976
|
}
|
|
1921
1977
|
|
|
1922
|
-
/// One copy's fragments (an exact prefix and suffix split by a large edit) as one occurrence
|
|
1923
|
-
/// mirrors coalesceOccurrences in duplication.ts.
|
|
1978
|
+
/// One copy's fragments (an exact prefix and suffix split by a large edit) as one occurrence.
|
|
1924
1979
|
fn coalesce_occurrences(occurrences: Vec<CountedOccurrence>) -> CountedOccurrence {
|
|
1925
1980
|
if occurrences.len() == 1 {
|
|
1926
1981
|
return occurrences.into_iter().next().expect("non-empty");
|
|
@@ -1966,8 +2021,8 @@ struct NormalizedBlock {
|
|
|
1966
2021
|
content_total: usize,
|
|
1967
2022
|
}
|
|
1968
2023
|
|
|
1969
|
-
/// A block's tokens as comparable integers
|
|
1970
|
-
///
|
|
2024
|
+
/// A block's tokens as comparable integers (literal VALUES are folded into the symbol, unlike the
|
|
2025
|
+
/// exact fingerprint's kind tags).
|
|
1971
2026
|
fn normalize_block_sequence(
|
|
1972
2027
|
tokens: &[Token<'_>],
|
|
1973
2028
|
range: &TokenRange,
|
package/native/src/functions.rs
CHANGED
|
@@ -1304,7 +1304,7 @@ fn declares_deduced_type(declarator: Node<'_>) -> bool {
|
|
|
1304
1304
|
.is_some_and(|declared| declared.kind() == "placeholder_type_specifier")
|
|
1305
1305
|
}
|
|
1306
1306
|
|
|
1307
|
-
/// Unwraps a C/C++ declarator chain to the declared name
|
|
1307
|
+
/// Unwraps a C/C++ declarator chain to the declared name.
|
|
1308
1308
|
fn unwrap_declarator_name(declarator: Option<Node<'_>>, code: &Source<'_>) -> Option<String> {
|
|
1309
1309
|
let mut current = declarator;
|
|
1310
1310
|
while let Some(node) = current {
|