spellr 0.1.0

data/lib/spellr/interactive.rb

@@ -0,0 +1,191 @@
+# frozen_string_literal: true
+
+require 'io/console'
+require 'readline'
+require_relative '../spellr'
+require_relative 'reporter'
+require_relative 'string_format'
+
+module Spellr
+  class Interactive # rubocop:disable Metrics/ClassLength
+    include Spellr::StringFormat
+
+    attr_reader :global_replacements, :global_skips
+    attr_reader :global_insensitive_replacements
+    attr_reader :global_insensitive_skips
+    attr_accessor :total_skipped
+    attr_accessor :total_fixed
+    attr_accessor :total_added
+
+    def finish(checked) # rubocop:disable Metrics/AbcSize
+      puts "\n"
+      puts "#{pluralize 'file', checked} checked"
+      puts "#{pluralize 'error', total} found"
+      puts "#{pluralize 'error', total_skipped} skipped" if total_skipped.positive?
+      puts "#{pluralize 'error', total_fixed} fixed" if total_fixed.positive?
+      puts "#{pluralize 'word', total_added} added" if total_added.positive?
+    end
+
+    def total
+      total_skipped + total_fixed + total_added
+    end
+
+    def initialize
+      @global_replacements = {}
+      @global_insensitive_replacements = {}
+      @global_skips = []
+      @global_insensitive_skips = []
+      @total_skipped = 0
+      @total_fixed = 0
+      @total_added = 0
+    end
+
+    def call(token)
+      return if attempt_global_replacement(token)
+      return if attempt_global_skip(token)
+
+      Spellr::Reporter.new.call(token)
+
+      prompt(token)
+    end
+
+    def prompt(token)
+      print bold('[a,s,S,r,R,e,?]')
+
+      handle_response(token)
+    rescue Interrupt
+      puts '^C again to exit'
+    end
+
+    def attempt_global_skip(token)
+      return unless global_skips.include?(token.to_s) ||
+        global_insensitive_skips.include?(token.normalize)
+
+      self.total_skipped += 1
+    end
+
+    def attempt_global_replacement(token)
+      global_replacement = global_replacements[token.to_s]
+      global_replacement ||= global_insensitive_replacements[token.normalize]
+      return unless global_replacement
+
+      token.replace(global_replacement)
+      self.total_fixed += 1
+      raise Spellr::DidReplacement, token
+    end
+
+    def clear_current_line
+      print "\r\e[K"
+    end
+
+    def handle_response(token) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
+      task = STDIN.getch
+      clear_current_line
+
+      case task
+      when "\u0003" # ctrl c
+        exit 1
+      when 'a'
+        handle_add(token)
+      when 's', "\u0004" # ctrl d
+        handle_skip(token)
+      when 'S'
+        handle_skip(token) { |skip_token| global_skips << skip_token.to_s }
+      when 'i'
+        handle_skip(token) { |skip_token| global_insensitive_skips << skip_token.downcase }
+      when 'R'
+        handle_replacement(token) { |replacement| global_replacements[token.to_s] = replacement }
+      when 'I'
+        handle_replacement(token) { |replacement| global_insensitive_replacements[token.normalize] = replacement }
+      when 'r'
+        handle_replacement(token)
+      when 'e'
+        handle_replace_line(token)
+      when '?'
+        handle_help(token)
+      else
+        clear_current_line
+        call(token)
+      end
+    end
+
+    def handle_skip(token)
+      self.total_skipped += 1
+      yield token if block_given?
+    end
+
+    # TODO: handle more than 16 options
+    def handle_add(token) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      puts "Add #{red(token)} to wordlist:"
+      wordlists = Spellr.config.languages_for(token.location.file).flat_map(&:addable_wordlists)
+
+      wordlists.each_with_index do |wordlist, i|
+        puts "[#{i.to_s(16)}] #{wordlist.name}"
+      end
+      choice = STDIN.getch
+      clear_current_line
+      case choice
+      when "\u0003" # ctrl c
+        puts '^C again to exit'
+        call(token)
+      when /\h/
+        wl = wordlists[choice.to_i(16)]
+        return handle_add(token) unless wl
+
+        wl.add(token)
+        self.total_added += 1
+        raise Spellr::DidAdd, token
+      else
+        handle_add(token)
+      end
+    end
+
+    def handle_replacement(token, original_token: token) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      readline_editable_print(token.chomp)
+      highlighted_token = token == original_token ? red(token) : token.highlight(original_token.char_range)
+      prompt = "#{aqua '>>'} #{highlighted_token.chomp}\n#{aqua '=>'} "
+      replacement = Readline.readline(prompt)
+      if replacement.empty?
+        call(token)
+      else
+        full_replacement = token == original_token ? replacement : replacement + "\n"
+        token.replace(full_replacement)
+        yield replacement if block_given?
+        self.total_fixed += 1
+        raise Spellr::DidReplacement, token
+      end
+    rescue Interrupt
+      puts '^C again to exit'
+      call(original_token)
+    end
+
+    def handle_replace_line(token)
+      handle_replacement(
+        token.line,
+        original_token: token
+      )
+    end
+
+    def handle_help(token) # rubocop:disable Metrics/AbcSize
+      puts "#{bold '[r]'} Replace #{red token}"
+      puts "#{bold '[R]'} Replace all future instances of #{red token}"
+      puts "#{bold '[s]'} Skip #{red token}"
+      puts "#{bold '[S]'} Skip all future instances of #{red token}"
+      puts "#{bold '[a]'} Add #{red token} to a word list"
+      puts "#{bold '[e]'} Edit the whole line"
+      puts "#{bold '[?]'} Show this help"
+      handle_response(token)
+    end
+
+    def readline_editable_print(string)
+      Readline.pre_input_hook = lambda {
+        Readline.refresh_line
+        Readline.insert_text string.to_s
+        Readline.redisplay
+
+        # Remove the hook right away.
+        Readline.pre_input_hook = nil
+      }
+    end
+  end
+end

data/lib/spellr/language.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+require_relative 'wordlist'
+
+module Spellr
+  class Language
+    attr_reader :name
+
+    def initialize(name, # rubocop:disable Metrics/ParameterLists
+      wordlists: [],
+      generate: nil,
+      only: [],
+      description: '',
+      hashbangs: [])
+      @name = name
+      @description = description
+      @generate = generate
+      @wordlist_paths = wordlists
+      @only = only
+      @hashbangs = hashbangs
+    end
+
+    def matches?(file)
+      return true if @only.empty?
+
+      file = Spellr::File.wrap(file)
+      return true if @only.any? { |o| file.fnmatch?(o) }
+      return true if file.hashbang && @hashbangs.any? { |h| file.hashbang.include?(h) }
+    end
+
+    def config_wordlists
+      @config_wordlists ||= @wordlist_paths.map(&Spellr::Wordlist.method(:new))
+    end
+
+    def all_wordlist_paths
+      @wordlist_paths + default_wordlists.map(&:path)
+    end
+
+    def wordlists
+      w = config_wordlists + default_wordlists.select(&:exist?)
+      return generate_wordlist if w.empty?
+
+      w
+    end
+
+    def generate_wordlist
+      return [] unless generate
+
+      require_relative 'cli'
+      require 'shellwords'
+      warn "Generating wordlist for #{name}"
+
+      Spellr::CLI.new(generate.shellsplit)
+
+      config_wordlists + default_wordlists
+    end
+
+    def addable_wordlists
+      ((config_wordlists - default_wordlists) + [project_wordlist]).uniq(&:path)
+    end
+
+    def gem_wordlist
+      @gem_wordlist ||= Spellr::Wordlist.new(
+        Pathname.new(__dir__).parent.parent.join('wordlists', "#{name}.txt")
+      )
+    end
+
+    def project_wordlist
+      @project_wordlist ||= Spellr::Wordlist.new(
+        Pathname.pwd.join('.spellr_wordlists', "#{name}.txt"),
+        name: name
+      )
+    end
+
+    def generated_project_wordlist
+      @generated_project_wordlist ||= Spellr::Wordlist.new(
+        Pathname.pwd.join('.spellr_wordlists', 'generated', "#{name}.txt")
+      )
+    end
+
+    private
+
+    attr_reader :generate
+
+    def load_wordlists(name, paths, _generate)
+      wordlists = paths + default_wordlist_paths(name)
+
+      wordlists.map(&Spellr::Wordlist.method(:new))
+    end
+
+    def custom_addable_wordlists(wordlists)
+      default_paths = default_wordlist_paths
+      wordlists.map { |w| Spellr::Wordlist.new(w) }.reject { |w| default_paths.include?(w.path) }
+    end
+
+    def default_wordlists
+      [
+        gem_wordlist,
+        generated_project_wordlist,
+        project_wordlist
+      ]
+    end
+  end
+end

data/lib/spellr/line_location.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Spellr
+  class LineLocation
+    attr_reader :file
+    attr_reader :line_number
+    attr_reader :char_offset
+    attr_reader :byte_offset
+
+    def initialize(file = '[String]', line_number = 1, char_offset: 0, byte_offset: 0)
+      @file = file
+      @line_number = line_number
+      @char_offset = char_offset
+      @byte_offset = byte_offset
+    end
+
+    def to_s
+      "#{relative_file_name}:#{line_number}"
+    end
+
+    def file_name
+      file.respond_to?(:to_path) ? file.to_path : file
+    end
+
+    def relative_file_name
+      Pathname.new(file_name).relative_path_from(Pathname.pwd)
+    end
+  end
+end

data/lib/spellr/line_tokenizer.rb

@@ -0,0 +1,181 @@
+# frozen_string_literal: true
+
+require 'strscan'
+require_relative '../spellr'
+require_relative 'column_location'
+require_relative 'token'
+
+module Spellr
+  class LineTokenizer < StringScanner # rubocop:disable Metrics/ClassLength
+    attr_reader :line
+    attr_accessor :disabled
+    alias_method :disabled?, :disabled
+    attr_accessor :skip_uri
+    alias_method :skip_uri?, :skip_uri
+    attr_accessor :skip_key
+    alias_method :skip_key?, :skip_key
+
+    def initialize(*line, skip_uri: true, skip_key: true)
+      @line = Spellr::Token.wrap(line.first)
+      @skip_uri = skip_uri
+      @skip_key = skip_key
+
+      super(@line.to_s)
+    end
+
+    def string=(line)
+      @line = Token.wrap(line)
+      super(@line.to_s)
+    end
+
+    def each_term
+      until eos?
+        term = next_term
+        next unless term
+        next if disabled?
+
+        yield term
+      end
+    end
+
+    def each_token
+      until eos?
+        term = next_term
+        next unless term
+        next if disabled?
+
+        yield Token.new(term, line: line, location: column_location(term))
+      end
+    end
+
+    private
+
+    def column_location(term)
+      ColumnLocation.new(
+        byte_offset: pos - term.bytesize,
+        char_offset: charpos - term.length,
+        line_location: line.location.line_location
+      )
+    end
+
+    def skip_nonwords_and_flags
+      skip_nonwords || skip_and_track_enable || skip_and_track_disable
+    end
+
+    def next_term
+      return if eos?
+
+      (skip_nonwords_and_flags && next_term) || scan_term || next_term
+    end
+
+    def scan_term
+      term = title_case || lower_case || upper_case || other_case
+
+      return term if term && term.length >= Spellr.config.word_minimum_length
+    end
+
+    NOT_EVEN_NON_WORDS_RE = %r{[^[:alpha:]/%#0-9\\]+}.freeze # everything not covered by more specific skips/scans
+    LEFTOVER_NON_WORD_BITS_RE = %r{[/%#0-9\\]}.freeze # e.g. a / not starting //a-url.com
+    HEX_RE = /(?:#(?:\h{6}|\h{3})|0x\h+)(?![[:alpha:]])/.freeze
+    SHELL_COLOR_ESCAPE_RE = /\\(e|033)\[\d+(;\d+)*m/.freeze
+    BACKSLASH_ESCAPE_RE = /\\[a-zA-Z]/.freeze # TODO: hex escapes e.g. \xAA. TODO: language aware escapes
+    REPEATED_SINGLE_LETTERS_RE = /(?:([[:alpha:]])\1+)(?![[:alpha:]])/.freeze # e.g. xxxxxxxx (it's not a word)
+    # https://developer.mozilla.org/en-US/docs/Glossary/percent-encoding
+    # Only the necessary percent encoding that actually ends in letters
+    # URL_ENCODED_ENTITIES_RE = /%(3A|2F|3F|5B|5D|%2A|%2B|%2C|%3B|%3D)/i.freeze
+    URL_ENCODED_ENTITIES_RE = /%[0-8A-F]{2}/.freeze
+    # There's got to be a better way of writing this
+    SEQUENTIAL_LETTERS_RE = /a(b(c(d(e(f(g(h(i(j(k(l(m(n(o(p(q(r(s(t(u(v(w(x(y(z)?)?)?)?)?)?)?)?)?)?)?)?)?)?)?)?)?)?)?)?)?)?)?)?)?(?![[:alpha:]])/i.freeze # rubocop:disable Metrics/LineLength
+
+    def skip_nonwords # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+      skip(NOT_EVEN_NON_WORDS_RE) ||
+        skip_uri_heuristically ||
+        skip_key_heuristically ||
+        skip(HEX_RE) ||
+        skip(URL_ENCODED_ENTITIES_RE) ||
+        skip(SHELL_COLOR_ESCAPE_RE) ||
+        skip(BACKSLASH_ESCAPE_RE) ||
+        skip(LEFTOVER_NON_WORD_BITS_RE) ||
+        skip(REPEATED_SINGLE_LETTERS_RE) ||
+        skip(SEQUENTIAL_LETTERS_RE)
+    end
+
+    # I didn't want to do this myself. BUT i need something to heuristically match on, and it's difficult
+    URL_RE = %r{
+      (?<scheme>//|https?://|s?ftp://|mailto:)?
+      (?<userinfo>[[:alnum:]]+(?::[[:alnum:]]+)?@)?
+      (?<hostname>(?:[[:alnum:]-]+(?:\\?\.[[:alnum:]-]+)+|localhost|\d{1,3}(?:.\d{1,3}){3}))
+      (?<port>:\d+)?
+      (?<path>/(?:[[:alnum:]=!$&\-/._\\]|%\h{2})+)?
+      (?<query>\?(?:[[:alnum:]=!$\-/.\\]|%\h{2})+(?:&(?:[[:alnum:]=!$\-/.\\]|%\h{2})+)*)?
+      (?<fragment>\#(?:[[:alnum:]=!$&\-/.\\]|%\h{2})+)?
+    }x.freeze
+    # unfortunately i have to match this regex a couple times because stringscanner doesn't give me matchdata
+    def skip_uri_heuristically
+      return unless skip_uri?
+      return unless match?(URL_RE)
+
+      captures = URL_RE.match(matched).named_captures
+      skip(URL_RE) if captures['scheme'] || captures['userinfo'] || captures['path']
+    end
+
+    # url unsafe base64 or url safe base64
+    # TODO: character distribution heuristic
+    KEY_FULL_RE = %r{([A-Za-z\d+/]|[A-Za-z\d\-_])+[=.]*}.freeze
+    KEY_RE = %r{
+      (?:
+        [A-Za-z\-_+/=]+|
+        [\d\-_+/=]+
+      )
+    }x.freeze
+    def skip_key_heuristically
+      return unless skip_key?
+      return unless match?(KEY_FULL_RE)
+
+      # can't use regular captures because repeated capture groups don't
+      matches = matched.scan(KEY_RE)
+      return unless matches.length >= 3 # number chosen arbitrarily
+
+      skip(KEY_FULL_RE)
+    end
+
+    # jump to character-aware position
+    def charpos=(new_charpos)
+      skip(/.{#{new_charpos - charpos}}/m)
+    end
+
+    # [Word], [Word]Word [Word]'s [Wordn't]
+    TITLE_CASE_RE = /[[:upper:]][[:lower:]]+(?:['’][[:lower:]]+(?<!['’]s))*/.freeze
+    def title_case
+      scan(TITLE_CASE_RE)
+    end
+
+    # [word] [word]'s [wordn't]
+    LOWER_CASE_RE = /[[:lower:]]+(?:['’][[:lower:]]+(?<!['’]s))*/.freeze
+    def lower_case
+      scan(LOWER_CASE_RE)
+    end
+
+    # [WORD] [WORD]Word [WORDN'T] [WORD]'S [WORD]'s [WORD]s
+    UPPER_CASE_RE = /[[:upper:]]+(?:['’][[:upper:]]+(?<!['’][Ss]))*((?![[:lower:]])|(?=s(?![[:lower:]])))/.freeze
+    def upper_case
+      scan(UPPER_CASE_RE)
+    end
+
+    # for characters in [:alpha:] that aren't in [:lower:] or [:upper:] e.g. Arabic
+    OTHER_CASE_RE = /[[:alpha:]]+/.freeze
+    def other_case
+      scan(OTHER_CASE_RE)
+    end
+
+    SPELLR_DISABLE_RE = /spellr:disable/.freeze
+    def skip_and_track_disable
+      skip(SPELLR_DISABLE_RE) && self.disabled = true
+    end
+
+    SPELLR_ENABLE_RE = /spellr:enable/.freeze
+    def skip_and_track_enable
+      skip(SPELLR_ENABLE_RE) && self.disabled = false
+    end
+  end
+end