rubocop-spellcheck 0.1.0 → 0.2.0

data/lib/rubocop/spellcheck/rdoc.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+require 'rdoc'
+require 'rdoc/markup'
+require 'rdoc/markup/inline_parser'
+
+module RuboCop
+  module Spellcheck
+    # Reads RDoc prose without checking verbatim examples or inline code.
+    class RDoc
+      # Records the original tokens consumed by paragraph and heading parsing.
+      class Parser < ::RDoc::Markup::Parser
+        attr_reader :prose
+
+        def initialize
+          super
+          @prose = {}
+        end
+
+        def build_paragraph(margin)
+          @margin = margin
+          super
+        ensure
+          @margin = nil
+        end
+
+        def build_heading(level)
+          @heading = true
+          super
+        ensure
+          @heading = false
+        end
+
+        def get
+          token = super
+          type, text, column, line = token
+          @prose[[line, column]] = text.dup if type == :TEXT && (@heading || column == @margin)
+          token
+        end
+      end
+
+      # Uses consumed inline token ranges to retain original source positions.
+      class Inline < ::RDoc::Markup::InlineParser
+        def initialize(source)
+          super
+          @source = source
+          @masked = source.dup
+        end
+
+        def prose
+          parse
+          @masked.gsub(/(?:link:|rdoc-(?:ref|label|image):)\S+/) { |link| ' ' * link.length }
+        end
+
+        private
+
+        def scan_token
+          first = @scanner.pos
+          result = super
+          raw = @source.byteslice(first...@scanner.pos)
+          offset = @source.byteslice(0...first).length
+          mask_token(result, raw, offset)
+          result
+        end
+
+        def mask_token(result, raw, offset)
+          type, _token, node = result
+          case type
+          when :node
+            mask_node(node, raw, offset)
+          when :open, :close, :tidylink_open, :tidylink_close, :invalidated_tidylink_close
+            mask(offset, raw.length)
+          end
+        end
+
+        def mask_node(node, raw, offset)
+          return mask(offset, raw.length) if %i[TT HARD_BREAK].include?(node[:type])
+
+          text = node[:children].join
+          first = raw.index(text)
+          mask(offset, first)
+          mask(offset + first + text.length, raw.length - first - text.length)
+        end
+
+        def mask(first, length)
+          @masked[first, length] = @masked[first, length].gsub(/[^\n]/, ' ')
+        end
+      end
+
+      def self.file?(path)
+        path && File.extname(path).casecmp?('.rdoc')
+      end
+
+      def initialize(buffer, source: buffer.source, positions: nil)
+        @buffer = buffer
+        @source = source
+        @positions = positions || source.length.times.map { |index| [index, index + 1] }
+      end
+
+      def each_span
+        return enum_for(__method__) unless block_given?
+
+        yield Span.new(Inline.new(prose_source).prose, @positions)
+      rescue ::RDoc::Markup::Parser::ParseError => e
+        raise RuboCop::ValidationError, "Invalid RDoc in #{@buffer.name}: #{e.message}"
+      end
+
+      private
+
+      def prose_source
+        source = @source.gsub(/[^\n]/, ' ')
+        offsets = line_offsets
+        prose.each do |(line, column), text|
+          source[offsets.fetch(line) + column, text.length] = text
+        end
+        source
+      end
+
+      def prose
+        parser = Parser.new
+        parser.tokenize(mask_directives)
+        parser.parse(::RDoc::Markup::Document.new)
+        parser.prose
+      end
+
+      def mask_directives
+        @source.gsub(/^[ \t]*:[a-z][\w-]*:.*$/i) { |text| ' ' * text.length }
+      end
+
+      def line_offsets
+        offset = 0
+        @source.lines.map do |line|
+          first = offset
+          offset += line.length
+          first
+        end
+      end
+    end
+  end
+end

data/lib/rubocop/spellcheck/rdoc_comments.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module RuboCop
+  module Spellcheck
+    # Combines adjacent Ruby comment lines before parsing their RDoc structure.
+    class RDocComments
+      def initialize(buffer, comments)
+        @buffer = buffer
+        @comments = comments
+      end
+
+      def each_span(&block)
+        return enum_for(__method__) unless block_given?
+
+        groups.each do |comments|
+          source, positions = comment_source(comments)
+          RDoc.new(@buffer, source: source, positions: positions).each_span(&block)
+        end
+      end
+
+      private
+
+      def groups
+        @comments.slice_when do |previous, current|
+          current.loc.line != previous.loc.last_line + 1 || current.text.start_with?('=begin') ||
+            previous.text.start_with?('=begin')
+        end
+      end
+
+      def comment_source(comments)
+        source = +''
+        positions = []
+        comments.each do |comment|
+          text, offsets = mapped_comment(comment)
+          source << text
+          positions.concat(offsets)
+          source << "\n"
+          positions << [comment.source_range.end_pos, comment.source_range.end_pos]
+        end
+        [source, positions]
+      end
+
+      def mapped_comment(comment)
+        first = comment.source_range.begin_pos
+        pieces = comment.text.lines.map do |line|
+          text, removed = strip_marker(line)
+          offsets = positions(first + removed, text.length)
+          first += line.length
+          [text, offsets]
+        end
+        [pieces.map(&:first).join, pieces.flat_map(&:last)]
+      end
+
+      def positions(first, length)
+        length.times.map { |index| [first + index, first + index + 1] }
+      end
+
+      def strip_marker(line)
+        return [line.gsub(/[^\n]/, ' '), 0] if line.match?(/\A=(?:begin|end)\b/)
+
+        marker = line[/\A# ?/].to_s.length
+        [line[marker..], marker]
+      end
+    end
+  end
+end

data/lib/rubocop/spellcheck/ruby_code.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module RuboCop
+  module Spellcheck
+    # Applies independent Ruby scope settings to source-aware extractors.
+    class RubyCode
+      DIRECTIVE = /
+        \A\#\s*(?:!|(?:rubocop|standard|coding|encoding|frozen_string_literal|
+        warn_indent|shareable_constant_value|typed)\s*:)|\A\#.*-\*-\s*(?:coding|encoding):
+      /x
+
+      def initialize(source, config: {})
+        valid = config.is_a?(Hash) &&
+                %w[Strings Identifiers].all? { |key| [true, false].include?(config.fetch(key, true)) }
+        unless valid
+          raise RuboCop::ValidationError, 'Spellcheck/Spelling Ruby Strings and Identifiers must be true or false.'
+        end
+
+        @source = source
+        @config = config
+      end
+
+      def each_span(&)
+        comments = @source.comments.reject { |comment| DIRECTIVE.match?(comment.text) }
+        RDocComments.new(@source.buffer, comments).each_span(&)
+        each_code_span(&)
+      end
+
+      def each_code_span(&)
+        return unless @source.ast
+
+        RubyStrings.new(@source).each_span(&) if @config.fetch('Strings', true)
+        RubyIdentifiers.new(@source).each_span(&) if @config.fetch('Identifiers', false)
+      end
+    end
+  end
+end

data/lib/rubocop/spellcheck/ruby_identifiers.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module RuboCop
+  module Spellcheck
+    # Splits Ruby names without treating symbols, keywords, or operators as identifiers.
+    class RubyIdentifiers
+      # Identifier occurrences cannot be renamed independently of their references.
+      class Span < Spellcheck::Span
+        def correction(word, suggestions)
+          Correction.new(word, suggestions)
+        end
+      end
+
+      # Keeps diagnostics and suggestions without offering identifier replacements.
+      class Correction < Spellcheck::Correction
+        def replacement; end
+      end
+
+      TYPES = %i[tIDENTIFIER tFID tCONSTANT tIVAR tCVAR tGVAR].freeze
+      BOUNDARY = /(?<=\p{Ll})(?=\p{Lu})|(?<=\p{Lu})(?=\p{Lu}\p{Ll})/
+
+      def initialize(source)
+        @source = source
+        @parameters = source.ast.each_node(:kwarg, :kwoptarg).to_set { |node| node.loc.name.begin_pos }
+        @tokens = source.tokens.select { |token| identifier?(token) }
+      end
+
+      def each_span(&block)
+        return enum_for(__method__) unless block_given?
+
+        @tokens.each do |token|
+          token.text.to_enum(:scan, /[\p{L}\p{M}]+/).each do
+            match = Regexp.last_match
+            split(match[0], token.pos.begin_pos + match.begin(0), &block)
+          end
+        end
+      end
+
+      private
+
+      def identifier?(token)
+        TYPES.include?(token.type) || (token.type == :tLABEL && @parameters.include?(token.pos.begin_pos))
+      end
+
+      def split(name, first)
+        name.split(BOUNDARY).each do |word|
+          positions = word.length.times.map { |index| [first + index, first + index + 1] }
+          yield Span.new(word, positions)
+          first += word.length
+        end
+      end
+    end
+  end
+end

data/lib/rubocop/spellcheck/ruby_string_map.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module RuboCop
+  module Spellcheck
+    # Uses Ruby's parser, never evaluation, to decode escape sequences and retain source ranges.
+    class RubyStringMap
+      ESCAPE = /\\(?:u\{[0-9a-fA-F\s]+\}|u[0-9a-fA-F]{4}|x[0-9a-fA-F]{1,2}|
+                   [0-7]{1,3}|(?:M-|C-|c)(?:\\(?:M-|C-|c))*\\?.|.)/mx
+      PIECE = /#{ESCAPE}|./m
+
+      def initialize(version)
+        @version = version
+        @escapes = {}
+      end
+
+      def positions(raw, first, expected, single:)
+        decoded, bytes = decoded_bytes(raw, first, single)
+        bytes = without_indentation(decoded, bytes, expected.b)
+        character_positions(expected, bytes)
+      end
+
+      private
+
+      def decoded_bytes(raw, first, single)
+        decoded = +''.b
+        bytes = []
+        raw.scan(pattern(single)) do |piece|
+          match = Regexp.last_match
+          piece = decode(piece, single)
+          decoded << piece.b
+          piece.bytesize.times { bytes << [first + match.begin(0), first + match.end(0)] }
+        end
+        [decoded, bytes]
+      end
+
+      def pattern(single)
+        return PIECE unless single
+        return /./m if single.empty?
+
+        /\\[#{Regexp.escape(single)}]|./m
+      end
+
+      def decode(piece, single)
+        return piece unless piece.start_with?('\\') && piece.length > 1
+        return piece.delete_prefix('\\') if single&.include?(piece[1])
+        return piece if single
+
+        @escapes[piece] ||= parsed_escape(piece)
+      end
+
+      def parsed_escape(piece)
+        source = RuboCop::ProcessedSource.new("\"#{piece}\"", @version)
+        return source.ast.value if source.valid_syntax? && source.ast&.str_type?
+
+        raise RuboCop::ValidationError, "Cannot map Ruby string escape #{piece.inspect}."
+      end
+
+      def without_indentation(decoded, bytes, expected)
+        return bytes if decoded == expected
+
+        offset = 0
+        lines = expected.lines
+        decoded.lines.flat_map.with_index do |line, index|
+          target = lines.fetch(index, '')
+          removed = indentation(line, target)
+          mapped = bytes.slice(offset + removed, target.bytesize)
+          offset += line.bytesize
+          mapped
+        end
+      end
+
+      def indentation(line, target)
+        removed = line.bytesize - target.bytesize
+        return removed if removed >= 0 && line.end_with?(target) && line.byteslice(0, removed).match?(/\A[ \t]*\z/)
+
+        raise RuboCop::ValidationError, "Cannot map decoded Ruby string #{target.inspect}."
+      end
+
+      def character_positions(text, bytes)
+        offset = 0
+        text.each_char.map do |character|
+          range = [bytes.fetch(offset).first, bytes.fetch(offset + character.bytesize - 1).last]
+          offset += character.bytesize
+          range
+        end
+      end
+    end
+  end
+end

data/lib/rubocop/spellcheck/ruby_strings.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+module RuboCop
+  module Spellcheck
+    # Extracts string tokens selected by the syntax tree, excluding symbols and executable strings.
+    class RubyStrings
+      PATH_ARGUMENTS = { require: 0, require_relative: 0, load: 0, autoload: 1 }.freeze
+
+      # Retains delimiters and heredoc boundaries for source-safe replacement.
+      class Span < Spellcheck::Span
+        def initialize(text, positions, delimiters, terminator)
+          super(text, positions)
+          @delimiters = delimiters
+          @terminator = terminator
+        end
+
+        def correction(word, suggestions)
+          Correction.new(word, suggestions, @delimiters, @terminator)
+        end
+      end
+
+      # Escapes replacement delimiters and never introduces a heredoc terminator.
+      class Correction < Spellcheck::Correction
+        def initialize(word, suggestions, delimiters, terminator)
+          super(word, suggestions)
+          @delimiters = delimiters
+          @terminator = terminator
+        end
+
+        def replacement
+          result = super
+          return unless result && result != @terminator
+
+          result.each_char.map { |character| @delimiters.include?(character) ? "\\#{character}" : character }.join
+        end
+      end
+
+      def initialize(source)
+        @source = source
+        @map = RubyStringMap.new(source.ruby_version)
+        @tokens = source.tokens.select { |token| %i[tSTRING tSTRING_CONTENT tCHARACTER].include?(token.type) }
+                        .sort_by { |token| token.pos.begin_pos }
+      end
+
+      def each_span
+        return enum_for(__method__) unless block_given?
+
+        @source.ast.each_node(:str) do |node|
+          next if excluded?(node)
+
+          range = content_range(node)
+          tokens_in(range).each { |token| yield mapped_span(node, token, range) }
+        end
+      end
+
+      private
+
+      def mapped_span(node, token, range)
+        raw = token.type == :tSTRING_CONTENT ? token.pos : range
+        positions = @map.positions(raw.source, raw.begin_pos, token.text, single: single_escapes(node))
+        text = token.text.each_char.map { |character| character.valid_encoding? ? character : ' ' }.join
+        Span.new(text, positions, delimiters(node), terminator(node))
+      end
+
+      def tokens_in(range)
+        first = @tokens.bsearch_index { |token| token.pos.end_pos > range.begin_pos }
+        return [] unless first
+
+        last = @tokens.bsearch_index { |token| token.pos.begin_pos >= range.end_pos } || @tokens.length
+        @tokens[first...last]
+      end
+
+      def excluded?(node)
+        return true unless node.heredoc? || node.loc.respond_to?(:begin) # __FILE__ has no literal delimiters.
+        return true if dependency_string?(node)
+
+        node.each_ancestor do |ancestor|
+          return false if ancestor.begin_type?
+          return true if %i[regexp dsym xstr].include?(ancestor.type)
+        end
+        false
+      end
+
+      def dependency_string?(node)
+        node = node.parent while node.parent && %i[dstr array].include?(node.parent.type)
+        call = node.parent
+        return false unless call&.send_type?
+
+        dependency_argument?(call, node) &&
+          (call.method?(:autoload) ? module_receiver?(call.receiver) : kernel_receiver?(call.receiver))
+      end
+
+      def dependency_argument?(call, node)
+        return call.arguments.include?(node) if call.method?(:gem)
+        return false unless PATH_ARGUMENTS.key?(call.method_name)
+
+        call.arguments[PATH_ARGUMENTS.fetch(call.method_name)].equal?(node)
+      end
+
+      def module_receiver?(receiver)
+        receiver.nil? || receiver.self_type? || receiver.const_type?
+      end
+
+      def kernel_receiver?(receiver)
+        receiver.nil? || receiver.self_type? ||
+          (receiver.const_type? && %w[Kernel ::Kernel].include?(receiver.const_name))
+      end
+
+      def content_range(node)
+        return node.loc.heredoc_body if node.heredoc?
+
+        location = node.loc
+        range = location.expression
+        range.with(begin_pos: location.begin&.end_pos || range.begin_pos,
+                   end_pos: location.end&.begin_pos || range.end_pos)
+      end
+
+      def single_escapes(node)
+        opening = literal_opening(node)
+        return '' if opening.match?(/\A<<[-~]?'/)
+        return "\\'" if opening == "'"
+        return unless opening.start_with?('%q', '%w')
+
+        whitespace = opening.start_with?('%w') ? " \t\r\n" : ''
+        "\\#{whitespace}#{paired_delimiters(opening[-1])}"
+      end
+
+      def delimiters(node)
+        opening = literal_opening(node)
+        return '' if opening.start_with?('<<')
+
+        paired_delimiters(opening[-1])
+      end
+
+      def paired_delimiters(delimiter)
+        return '' unless delimiter
+
+        "#{delimiter}#{{ '(' => ')', '[' => ']', '{' => '}', '<' => '>' }.fetch(delimiter, delimiter)}"
+      end
+
+      def terminator(node)
+        container = node.parent&.dstr_type? ? node.parent : node
+        container.loc.heredoc_end.source.strip if container.heredoc?
+      end
+
+      def literal_opening(node)
+        return node.loc.expression.source if node.heredoc?
+        return node.loc.begin.source if node.loc.begin
+
+        parent = node.parent
+        return literal_opening(parent) if parent&.dstr_type?
+
+        parent_opening(parent)
+      end
+
+      def parent_opening(parent)
+        return '' unless parent&.loc.respond_to?(:begin)
+
+        parent.loc.begin&.source || ''
+      end
+    end
+  end
+end

data/lib/rubocop/spellcheck/span.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module RuboCop
+  module Spellcheck
+    # Associates extracted prose with its original positions and correction policy.
+    class Span
+      attr_reader :text, :positions
+
+      def initialize(text, positions)
+        @text = text
+        @positions = positions
+      end
+
+      def correction(word, suggestions)
+        Correction.new(word, suggestions)
+      end
+
+      def each_misspelling(checker, text)
+        checker.misspellings(text).each do |word, position|
+          next unless word.match?(/\p{L}/)
+
+          whole = Correction.whole_word?(positions, position, word.length) &&
+                  Correction.standalone?(text, position, word.length)
+          yield word, *range(position, word.length), correction(word, checker.suggestions(word)), whole
+        end
+      end
+
+      private
+
+      def range(position, length)
+        [positions.fetch(position).first, positions.fetch(position + length - 1).last]
+      end
+    end
+  end
+end

data/lib/rubocop/spellcheck/version.rb

@@ -2,6 +2,6 @@
 
 module RuboCop
   module Spellcheck
-    VERSION = '0.1.0'
+    VERSION = '0.2.0'
   end
 end

data/lib/rubocop/spellcheck/vocabulary.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module RuboCop
+  module Spellcheck
+    # Rebuilds vocabulary from current violations while retaining comments and needed word spelling.
+    class Vocabulary
+      # Still fingerprints and excludes the vocabulary file, but does not accept its words.
+      class Checker < Spellcheck::Checker
+        def suggestions(_word)
+          # Vocabulary generation only collects words, so skip expensive suggestions.
+          []
+        end
+
+        private
+
+        def dictionary_word(line, path)
+          super unless path == File.join(@base_directory, '.spellcheck.txt')
+        end
+      end
+
+      def initialize(config)
+        @config = config
+        @path = File.join(config.base_dir_for_path_parameters, '.spellcheck.txt')
+        @words = Set.new
+      end
+
+      def add(offense)
+        word = offense.message[/Misspelled word: `([^`]+)`\./, 1]
+        unless word
+          raise RuboCop::ValidationError, "Cannot extract spelling vocabulary from #{offense.message.inspect}."
+        end
+
+        @words << word.downcase
+      end
+
+      def write
+        content = File.exist?(@path) ? File.read(@path, encoding: 'UTF-8') : ''
+        additions = (@words - existing_words(content)).sort
+        config = VocabularyConfig.new(@config.loaded_path, section: section).updated(dictionaries)
+        updated = extended(retained(content), additions)
+        File.write(@path, updated) unless content == updated
+        write_configuration(config)
+        puts "Updated #{RuboCop::PathUtil.relative_path(@path)}. " \
+             'Review generated vocabulary: reported typos are accepted too.'
+      end
+
+      private
+
+      def retained(content)
+        content.lines.select do |line|
+          word = line.strip
+          word.empty? || word.start_with?('#') || @words.include?(word.downcase)
+        end.join
+      end
+
+      def write_configuration(content)
+        File.write(@config.loaded_path, content) unless File.read(@config.loaded_path, encoding: 'UTF-8') == content
+      end
+
+      def dictionaries
+        badge = RuboCop::Cop::Badge.parse(VocabularyConfig::COP)
+        files = @config.for_badge(badge)['DictionaryFiles'] || Checker::DEFAULT_FILES
+        files.transform_values do |paths|
+          paths.any? { |path| vocabulary_path?(path) } ? paths : [*paths, '.spellcheck.txt']
+        end
+      end
+
+      def vocabulary_path?(path)
+        !path.start_with?('gem:') && File.expand_path(path, @config.base_dir_for_path_parameters) == @path
+      end
+
+      def section
+        @config.for_cop(VocabularyConfig::COP).key?('DictionaryFiles') ? VocabularyConfig::COP : 'Spellcheck'
+      end
+
+      def existing_words(content)
+        content.lines.filter_map do |line|
+          word = line.strip
+          next if word.empty? || word.start_with?('#')
+          raise RuboCop::ValidationError, "Expected one word per line in #{@path}: #{word.inspect}" if word.match?(/\s/)
+
+          word.downcase
+        end.to_set
+      end
+
+      def extended(content, additions)
+        return content if additions.empty?
+
+        separator = content.empty? || content.end_with?("\n") ? '' : "\n"
+        "#{content}#{separator}#{additions.join("\n")}\n"
+      end
+    end
+  end
+end