jscpd-rs 0.1.0

package/src/tokenizer/oxc/fallback.rs

@@ -0,0 +1,275 @@
+use super::super::scan::{scan_block_comment, scan_line_comment};
+use super::super::{
+    ByteSpan, DetectionToken, LineIndex, TokenContext, TokenKind, push_strict_whitespace_tokens,
+    push_token,
+};
+use super::lexical::{is_js_constant, is_js_keyword};
+use super::push_line_comment_tokens;
+use super::scan_regex_literal_end;
+
+pub(super) fn tokenize_js_like_range(
+    tokens: &mut Vec<DetectionToken>,
+    context: &TokenContext<'_>,
+    range_start: usize,
+    range_end: usize,
+    line_index: &LineIndex,
+) {
+    let bytes = context.content.as_bytes();
+    let mut idx = range_start;
+
+    while idx < range_end {
+        let ch = context.content[idx..].chars().next().unwrap_or('\0');
+        if ch.is_whitespace() {
+            let whitespace_end = scan_whitespace(context.content, idx, range_end);
+            push_strict_whitespace_tokens(
+                tokens,
+                context,
+                ByteSpan {
+                    start: idx,
+                    end: whitespace_end,
+                },
+                line_index,
+            );
+            idx = whitespace_end.max(idx + ch.len_utf8());
+            continue;
+        }
+
+        if idx + 1 < range_end && bytes[idx] == b'/' && bytes[idx + 1] == b'/' {
+            let end = scan_line_comment(bytes, idx, range_end);
+            if context.options.mode != crate::cli::Mode::Weak {
+                push_line_comment_tokens(tokens, context, ByteSpan { start: idx, end }, line_index);
+            }
+            idx = end.max(idx + 1);
+            continue;
+        }
+
+        let (end, kind) = if idx + 1 < range_end && bytes[idx] == b'/' && bytes[idx + 1] == b'*' {
+            (
+                scan_block_comment(bytes, idx, range_end),
+                TokenKind::Comment,
+            )
+        } else if bytes[idx] == b'/' {
+            if let Some(end) = scan_regex_literal_end(context.content, idx, range_end) {
+                (end, TokenKind::String)
+            } else {
+                scan_operator_or_punctuation(bytes, idx, range_end)
+            }
+        } else if bytes[idx] == b'`' {
+            (
+                scan_template_literal(bytes, idx, range_end).unwrap_or(range_end),
+                TokenKind::String,
+            )
+        } else if matches!(bytes[idx], b'\'' | b'"') {
+            if let Some(end) = scan_closed_string(bytes, idx, bytes[idx], range_end) {
+                (end, TokenKind::String)
+            } else {
+                (
+                    scan_unclosed_quote_fragment(context.content, idx, range_end),
+                    TokenKind::Default,
+                )
+            }
+        } else if is_identifier_start(ch) {
+            let end = scan_identifier(context.content, idx, range_end);
+            let value = &context.content[idx..end];
+            let kind = if is_js_constant(value) {
+                TokenKind::Constant
+            } else if is_js_keyword(value) {
+                TokenKind::Keyword
+            } else {
+                TokenKind::Default
+            };
+            (end, kind)
+        } else if bytes[idx].is_ascii_digit() {
+            (scan_number(bytes, idx, range_end), TokenKind::Number)
+        } else {
+            scan_operator_or_punctuation(bytes, idx, range_end)
+        };
+
+        push_token(
+            tokens,
+            context,
+            kind,
+            ByteSpan { start: idx, end },
+            line_index.location(idx),
+            line_index.location(end),
+        );
+        idx = end.max(idx + 1);
+    }
+}
+
+fn scan_closed_string(bytes: &[u8], start: usize, quote: u8, limit: usize) -> Option<usize> {
+    let mut idx = start + 1;
+    while idx < limit {
+        if bytes[idx] == b'\\' {
+            idx = (idx + 2).min(limit);
+            continue;
+        }
+        if matches!(bytes[idx], b'\n' | b'\r') {
+            return None;
+        }
+        if bytes[idx] == quote {
+            return Some(idx + 1);
+        }
+        idx += 1;
+    }
+    None
+}
+
+fn scan_template_literal(bytes: &[u8], start: usize, limit: usize) -> Option<usize> {
+    let mut idx = start + 1;
+    while idx < limit {
+        if bytes[idx] == b'\\' {
+            idx = (idx + 2).min(limit);
+            continue;
+        }
+        if bytes[idx] == b'`' {
+            return Some(idx + 1);
+        }
+        idx += 1;
+    }
+    None
+}
+
+fn scan_unclosed_quote_fragment(content: &str, start: usize, limit: usize) -> usize {
+    let bytes = content.as_bytes();
+    let mut idx = start + 1;
+    while idx < limit {
+        let ch = content[idx..].chars().next().unwrap_or('\0');
+        if ch.is_whitespace() || is_js_text_delimiter(bytes[idx]) {
+            break;
+        }
+        idx += ch.len_utf8();
+    }
+    idx
+}
+
+fn scan_whitespace(content: &str, start: usize, limit: usize) -> usize {
+    let mut end = start;
+    while end < limit {
+        let ch = content[end..].chars().next().unwrap_or('\0');
+        if !ch.is_whitespace() {
+            break;
+        }
+        end += ch.len_utf8();
+    }
+    end
+}
+
+fn scan_identifier(content: &str, start: usize, limit: usize) -> usize {
+    let mut idx = start;
+    while idx < limit {
+        let ch = content[idx..].chars().next().unwrap_or('\0');
+        if !is_identifier_continue(ch) {
+            break;
+        }
+        idx += ch.len_utf8();
+    }
+    idx
+}
+
+fn scan_number(bytes: &[u8], start: usize, limit: usize) -> usize {
+    let mut idx = start;
+    while idx < limit
+        && (bytes[idx].is_ascii_alphanumeric() || matches!(bytes[idx], b'.' | b'_' | b'+' | b'-'))
+    {
+        idx += 1;
+    }
+    idx
+}
+
+fn scan_operator_or_punctuation(bytes: &[u8], start: usize, limit: usize) -> (usize, TokenKind) {
+    const OPERATORS: &[&[u8]] = &[
+        b">>>=", b"===", b"!==", b">>>", b"<<=", b">>=", b"**=", b"=>", b"==", b"!=", b"<=", b">=",
+        b"++", b"--", b"&&", b"||", b"??", b"?.", b"...", b"+=", b"-=", b"*=", b"/=", b"%=", b"&=",
+        b"|=", b"^=", b"<<", b">>", b"**",
+    ];
+    for operator in OPERATORS {
+        if bytes[start..limit].starts_with(operator) {
+            return (start + operator.len(), TokenKind::Operator);
+        }
+    }
+    let kind = if matches!(
+        bytes[start],
+        b'{' | b'}' | b'[' | b']' | b'(' | b')' | b';' | b',' | b':' | b'.'
+    ) {
+        TokenKind::Punctuation
+    } else {
+        TokenKind::Operator
+    };
+    (start + 1, kind)
+}
+
+fn is_js_text_delimiter(byte: u8) -> bool {
+    matches!(
+        byte,
+        b'{' | b'}'
+            | b'['
+            | b']'
+            | b'('
+            | b')'
+            | b';'
+            | b','
+            | b':'
+            | b'.'
+            | b'<'
+            | b'>'
+            | b'='
+            | b'+'
+            | b'-'
+            | b'*'
+            | b'/'
+            | b'%'
+            | b'&'
+            | b'|'
+            | b'^'
+            | b'!'
+            | b'?'
+            | b'~'
+    )
+}
+
+fn is_identifier_start(ch: char) -> bool {
+    ch == '_' || ch == '$' || ch.is_ascii_alphabetic() || (ch as u32) > 0x7f
+}
+
+fn is_identifier_continue(ch: char) -> bool {
+    is_identifier_start(ch) || ch.is_ascii_digit()
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::cli::Options;
+    use crate::tokenizer::hash_token;
+
+    use super::*;
+
+    #[test]
+    fn multiline_template_literals_are_single_string_tokens() {
+        let content = "expect(store).toMatchInlineSnapshot(`\n  [root]\n`);\n";
+        let options = Options::default();
+        let line_index = LineIndex::new(content);
+        let context = TokenContext {
+            content,
+            options: &options,
+            ignore_regions: &[],
+        };
+        let mut tokens = Vec::new();
+
+        tokenize_js_like_range(&mut tokens, &context, 0, content.len(), &line_index);
+
+        let template = tokens
+            .iter()
+            .find(|token| content[token.range[0]..token.range[1]].starts_with('`'))
+            .expect("template token");
+        assert_eq!(
+            &content[template.range[0]..template.range[1]],
+            "`\n  [root]\n`"
+        );
+        assert_eq!(template.start.line, 1);
+        assert_eq!(template.end.line, 3);
+        assert_eq!(
+            template.hash,
+            hash_token(TokenKind::String, "`\n  [root]\n`", false)
+        );
+    }
+}

package/src/tokenizer/oxc/jsx.rs

@@ -0,0 +1,168 @@
+use oxc_parser::Kind;
+
+use super::super::scan::count_prism_whitespace_tokens;
+use super::super::{ByteSpan, DetectionToken, LineIndex, TokenContext, TokenKind};
+use super::{RawOxcToken, push_oxc_token, push_token_part};
+
+pub(super) fn tokenize_jsx_attribute_scripts(
+    parser_tokens: &[RawOxcToken],
+    groups: &[(usize, usize)],
+    context: &TokenContext<'_>,
+    line_index: &LineIndex,
+) -> Vec<DetectionToken> {
+    let mut tokens = Vec::new();
+    let mut next_position = 0usize;
+    let mut previous_group_end = None;
+
+    for &(group_start_idx, group_end_idx) in groups {
+        let group_start = parser_tokens[group_start_idx].span.start;
+        if let Some(previous_end) = previous_group_end {
+            next_position += count_embedded_gap_positions(
+                context.content,
+                parser_tokens,
+                previous_end,
+                group_start,
+            );
+        }
+        let mut expression_depth = 0usize;
+        let mut previous_token_end = None;
+        for raw in &parser_tokens[group_start_idx..=group_end_idx] {
+            let before = tokens.len();
+            // Prism keeps default whitespace string tokens inside nested JSX
+            // script objects, and those tokens can decide minTokens windows.
+            if expression_depth >= 2
+                && let Some(gap_start) = previous_token_end
+            {
+                push_embedded_default_gap(
+                    &mut tokens,
+                    context,
+                    gap_start,
+                    raw.span.start,
+                    line_index,
+                );
+            }
+            push_oxc_token(&mut tokens, context, raw.kind, raw.span, line_index);
+            for pushed in &mut tokens[before..] {
+                pushed.start.position = next_position;
+                pushed.end.position = next_position;
+                next_position += 1;
+            }
+            match raw.kind {
+                Kind::LCurly => expression_depth += 1,
+                Kind::RCurly => expression_depth = expression_depth.saturating_sub(1),
+                _ => {}
+            }
+            previous_token_end = Some(raw.span.end);
+        }
+        previous_group_end = Some(parser_tokens[group_end_idx].span.end);
+    }
+
+    tokens
+}
+
+fn push_embedded_default_gap(
+    tokens: &mut Vec<DetectionToken>,
+    context: &TokenContext<'_>,
+    gap_start: usize,
+    gap_end: usize,
+    line_index: &LineIndex,
+) {
+    if gap_start >= gap_end {
+        return;
+    }
+    if !context.content[gap_start..gap_end]
+        .chars()
+        .all(char::is_whitespace)
+    {
+        return;
+    }
+    push_token_part(
+        tokens,
+        context,
+        TokenKind::Default,
+        ByteSpan {
+            start: gap_start,
+            end: gap_end,
+        },
+        line_index,
+    );
+}
+
+pub(super) fn jsx_attribute_script_groups(parser_tokens: &[RawOxcToken]) -> Vec<(usize, usize)> {
+    let mut groups = Vec::new();
+    let mut in_jsx_tag = false;
+    let mut idx = 0usize;
+
+    while idx < parser_tokens.len() {
+        let token = parser_tokens[idx];
+        if !in_jsx_tag && token.kind == Kind::LAngle && looks_like_jsx_tag_start(parser_tokens, idx)
+        {
+            in_jsx_tag = true;
+            idx += 1;
+            continue;
+        }
+        if in_jsx_tag && token.kind == Kind::RAngle {
+            in_jsx_tag = false;
+            idx += 1;
+            continue;
+        }
+        if in_jsx_tag
+            && token.kind == Kind::Eq
+            && parser_tokens
+                .get(idx + 1)
+                .is_some_and(|next| next.kind == Kind::LCurly)
+            && let Some(group_end_idx) = jsx_attribute_expression_end(parser_tokens, idx + 1)
+        {
+            groups.push((idx, group_end_idx));
+            idx = group_end_idx + 1;
+            continue;
+        }
+        idx += 1;
+    }
+
+    groups
+}
+
+fn looks_like_jsx_tag_start(parser_tokens: &[RawOxcToken], idx: usize) -> bool {
+    matches!(
+        parser_tokens.get(idx + 1).map(|token| token.kind),
+        Some(Kind::Ident) | Some(Kind::This) | Some(Kind::PrivateIdentifier)
+    ) || matches!(
+        (
+            parser_tokens.get(idx + 1).map(|token| token.kind),
+            parser_tokens.get(idx + 2).map(|token| token.kind),
+        ),
+        (Some(Kind::Slash), Some(Kind::Ident))
+    )
+}
+
+fn jsx_attribute_expression_end(parser_tokens: &[RawOxcToken], lcurly_idx: usize) -> Option<usize> {
+    let mut depth = 0usize;
+    for (idx, token) in parser_tokens.iter().enumerate().skip(lcurly_idx) {
+        match token.kind {
+            Kind::LCurly => depth += 1,
+            Kind::RCurly => {
+                depth = depth.saturating_sub(1);
+                if depth == 0 {
+                    return Some(idx);
+                }
+            }
+            _ => {}
+        }
+    }
+    None
+}
+
+fn count_embedded_gap_positions(
+    content: &str,
+    parser_tokens: &[RawOxcToken],
+    gap_start: usize,
+    gap_end: usize,
+) -> usize {
+    count_prism_whitespace_tokens(content, gap_start, gap_end)
+        + parser_tokens
+            .iter()
+            .filter(|token| token.span.start >= gap_start && token.span.end <= gap_end)
+            .filter(|token| token.kind != Kind::Skip)
+            .count()
+}

package/src/tokenizer/oxc/kind.rs

@@ -0,0 +1,177 @@
+use oxc_parser::Kind;
+
+use super::super::TokenKind;
+use super::lexical::is_js_constant;
+
+pub(super) fn oxc_token_kind(kind: Kind, value: &str) -> TokenKind {
+    if kind == Kind::Ident && is_js_constant(value) {
+        TokenKind::Constant
+    } else {
+        token_kind_for_oxc(kind)
+    }
+}
+
+fn token_kind_for_oxc(kind: Kind) -> TokenKind {
+    if kind.is_number() {
+        return TokenKind::Number;
+    }
+    if matches!(
+        kind,
+        Kind::Str
+            | Kind::NoSubstitutionTemplate
+            | Kind::TemplateHead
+            | Kind::TemplateMiddle
+            | Kind::TemplateTail
+            | Kind::RegExp
+    ) {
+        return TokenKind::String;
+    }
+    if is_oxc_keyword(kind) {
+        return TokenKind::Keyword;
+    }
+    if is_oxc_punctuation(kind) {
+        return TokenKind::Punctuation;
+    }
+    if is_oxc_operator(kind) {
+        return TokenKind::Operator;
+    }
+    TokenKind::Default
+}
+
+fn is_oxc_keyword(kind: Kind) -> bool {
+    matches!(
+        kind,
+        Kind::Await
+            | Kind::Break
+            | Kind::Case
+            | Kind::Catch
+            | Kind::Class
+            | Kind::Const
+            | Kind::Continue
+            | Kind::Debugger
+            | Kind::Default
+            | Kind::Delete
+            | Kind::Do
+            | Kind::Else
+            | Kind::Enum
+            | Kind::Export
+            | Kind::Extends
+            | Kind::Finally
+            | Kind::For
+            | Kind::Function
+            | Kind::If
+            | Kind::Import
+            | Kind::In
+            | Kind::Instanceof
+            | Kind::New
+            | Kind::Return
+            | Kind::Super
+            | Kind::Switch
+            | Kind::This
+            | Kind::Throw
+            | Kind::Try
+            | Kind::Typeof
+            | Kind::Var
+            | Kind::Void
+            | Kind::While
+            | Kind::With
+            | Kind::Async
+            | Kind::From
+            | Kind::Get
+            | Kind::Of
+            | Kind::Set
+            | Kind::As
+            | Kind::Type
+            | Kind::Undefined
+            | Kind::Implements
+            | Kind::Interface
+            | Kind::Let
+            | Kind::Package
+            | Kind::Private
+            | Kind::Protected
+            | Kind::Public
+            | Kind::Static
+            | Kind::Yield
+            | Kind::True
+            | Kind::False
+            | Kind::Null
+    )
+}
+
+fn is_oxc_punctuation(kind: Kind) -> bool {
+    matches!(
+        kind,
+        Kind::Colon
+            | Kind::Comma
+            | Kind::Dot
+            | Kind::LBrack
+            | Kind::LCurly
+            | Kind::LParen
+            | Kind::RBrack
+            | Kind::RCurly
+            | Kind::RParen
+            | Kind::Semicolon
+    )
+}
+
+fn is_oxc_operator(kind: Kind) -> bool {
+    !matches!(kind, Kind::Ident | Kind::PrivateIdentifier | Kind::JSXText)
+        && !matches!(token_kind_for_operator_check(kind), TokenKind::Default)
+}
+
+fn token_kind_for_operator_check(kind: Kind) -> TokenKind {
+    if matches!(
+        kind,
+        Kind::Amp
+            | Kind::Amp2
+            | Kind::Amp2Eq
+            | Kind::AmpEq
+            | Kind::Bang
+            | Kind::Caret
+            | Kind::CaretEq
+            | Kind::Dot3
+            | Kind::Eq
+            | Kind::Eq2
+            | Kind::Eq3
+            | Kind::GtEq
+            | Kind::LAngle
+            | Kind::LtEq
+            | Kind::Minus
+            | Kind::Minus2
+            | Kind::MinusEq
+            | Kind::Neq
+            | Kind::Neq2
+            | Kind::Percent
+            | Kind::PercentEq
+            | Kind::Pipe
+            | Kind::Pipe2
+            | Kind::Pipe2Eq
+            | Kind::PipeEq
+            | Kind::Plus
+            | Kind::Plus2
+            | Kind::PlusEq
+            | Kind::Question
+            | Kind::Question2
+            | Kind::Question2Eq
+            | Kind::QuestionDot
+            | Kind::RAngle
+            | Kind::ShiftLeft
+            | Kind::ShiftLeftEq
+            | Kind::ShiftRight
+            | Kind::ShiftRight3
+            | Kind::ShiftRight3Eq
+            | Kind::ShiftRightEq
+            | Kind::Slash
+            | Kind::SlashEq
+            | Kind::Star
+            | Kind::Star2
+            | Kind::Star2Eq
+            | Kind::StarEq
+            | Kind::Tilde
+            | Kind::Arrow
+    ) {
+        TokenKind::Operator
+    } else {
+        TokenKind::Default
+    }
+}

package/src/tokenizer/oxc/lexical.rs

@@ -0,0 +1,67 @@
+pub(super) fn is_js_keyword(value: &str) -> bool {
+    matches!(
+        value,
+        "as" | "async"
+            | "await"
+            | "break"
+            | "case"
+            | "catch"
+            | "class"
+            | "const"
+            | "continue"
+            | "debugger"
+            | "default"
+            | "delete"
+            | "do"
+            | "else"
+            | "enum"
+            | "export"
+            | "extends"
+            | "false"
+            | "finally"
+            | "for"
+            | "from"
+            | "function"
+            | "get"
+            | "if"
+            | "implements"
+            | "import"
+            | "in"
+            | "instanceof"
+            | "interface"
+            | "let"
+            | "new"
+            | "null"
+            | "of"
+            | "package"
+            | "private"
+            | "protected"
+            | "public"
+            | "return"
+            | "set"
+            | "static"
+            | "super"
+            | "switch"
+            | "this"
+            | "throw"
+            | "true"
+            | "try"
+            | "type"
+            | "typeof"
+            | "undefined"
+            | "var"
+            | "void"
+            | "while"
+            | "with"
+            | "yield"
+    )
+}
+
+pub(super) fn is_js_constant(value: &str) -> bool {
+    let mut chars = value.chars();
+    let Some(first) = chars.next() else {
+        return false;
+    };
+    first.is_ascii_uppercase()
+        && chars.all(|ch| ch.is_ascii_uppercase() || ch.is_ascii_digit() || ch == '_')
+}