spellr 0.5.0 → 0.5.1

data/lib/spellr/key_tuner/stats.rb

@@ -1,6 +1,10 @@
 # frozen_string_literal: true
 
 module Stats
+  module_function
+
+  extend Math
+
   def mean(values, &block)
     return 0 if values.empty?
 
@@ -9,25 +13,40 @@ module Stats
 
   def min(values, &block)
     return 0 if values.empty?
+    return values.min unless block_given?
 
-    block ||= :itself.to_proc
-    block.call(values.min_by(&block))
+    yield values.min_by(&block)
   end
 
   def max(values, &block)
     return 0 if values.empty?
+    return values.max unless block_given?
 
-    block ||= :itself.to_proc
-    block.call(values.max_by(&block))
+    yield values.max_by(&block)
   end
 
-  def variance(values, &block)
+  def variance(values, &block) # rubocop:disable Metrics/MethodLength
     return 0 if values.empty?
 
-    values.sum { |sample| (mean(values, &block) - (block ? block.call(sample) : sample))**2 }.to_f / values.length
+    mean = mean(values, &block)
+    values.sum do |value|
+      value = yield value if block_given?
+      (mean - value)**2
+    end.to_f / values.length
   end
 
   def standard_deviation(values, &block)
-    Math.sqrt(variance(values, &block))
+    sqrt(variance(values, &block))
+  end
+
+  def gaussian_probability(value, standard_deviation:, mean:, variance:)
+    # deal with the edge case of a 0 standard deviation
+    if standard_deviation == 0
+      return mean == value ? 1.0 : 0.0
+    end
+
+    # calculate the gaussian probability
+    exp = -((value - mean)**2) / (2 * variance)
+    (1.0 / sqrt(2 * Math::PI * variance)) * (Math::E**exp)
   end
 end

data/lib/spellr/language.rb

@@ -7,52 +7,45 @@ module Spellr
     attr_reader :name
     attr_reader :key
 
-    def initialize(name, # rubocop:disable Metrics/ParameterLists, Metrics/MethodLength
-      key: name[0],
-      generate: nil,
-      only: [],
-      includes: [],
-      description: '',
-      hashbangs: [],
-      locale: [])
-      unless only.empty?
-        warn <<~WARNING
-          \e[33mSpellr: `only:` language yaml key with a list of fnmatch rules is deprecated.
-          Please use `includes:` instead, which uses gitignore-inspired rules.
-          see github.com/robotdana/fast_ignore#using-an-includes-list for details\e[0m
-        WARNING
-      end
-
-      if generate
-        warn <<~WARNING
-          \e[33mSpellr: `generate:` and generation is now deprecated. Choose the language
-          using the key `locale:` (any of US,AU,CA,GB,GBz,GBs as a string or array).\e[0m
-        WARNING
-      end
-
+    def initialize(name, key: name[0], includes: [], hashbangs: [], locale: [])
       @name = name
       @key = key
-      @description = description
-      @includes = only + includes
+      @includes = includes
       @hashbangs = hashbangs
       @locales = Array(locale)
     end
 
     def matches?(file)
-      return true if @includes.empty?
+      matches_includes?(file) || matches_hashbangs?(file)
+    end
 
-      return true if fast_ignore.allowed?(file.to_s)
+    def wordlists
+      default_wordlists.select(&:exist?)
+    end
 
-      file = Spellr::File.wrap(file)
-      return true if !@hashbangs.empty? && file.hashbang && @hashbangs.any? { |h| file.hashbang.include?(h) }
+    def project_wordlist
+      @project_wordlist ||= Spellr::Wordlist.new(
+        Pathname.pwd.join('.spellr_wordlists', "#{name}.txt"),
+        name: name
+      )
     end
 
-    def fast_ignore
-      @fast_ignore ||= FastIgnore.new(include_rules: @includes, gitignore: false)
+    private
+
+    def matches_hashbangs?(file)
+      return @includes.empty? if @hashbangs.empty?
+
+      file = Spellr::File.wrap(file)
+      return unless file.hashbang
+
+      @hashbangs.any? { |h| file.hashbang.include?(h) }
     end
 
-    def wordlists
-      default_wordlists.select(&:exist?)
+    def matches_includes?(file)
+      return @hashbangs.empty? if @includes.empty?
+
+      @fast_ignore ||= FastIgnore.new(include_rules: @includes, gitignore: false)
+      @fast_ignore.allowed?(file.to_s)
     end
 
     def gem_wordlist
@@ -61,13 +54,6 @@ module Spellr
       )
     end
 
-    def project_wordlist
-      @project_wordlist ||= Spellr::Wordlist.new(
-        Pathname.pwd.join('.spellr_wordlists', "#{name}.txt"),
-        name: name
-      )
-    end
-
     def locale_wordlists
       @locale_wordlists ||= @locales.map do |locale|
         Spellr::Wordlist.new(
@@ -76,8 +62,6 @@ module Spellr
       end
     end
 
-    private
-
     def load_wordlists(name, paths)
       wordlists = paths + default_wordlist_paths(name)
 
@@ -87,8 +71,8 @@ module Spellr
     def default_wordlists
       [
         gem_wordlist,
-        project_wordlist,
-        *locale_wordlists
+        *locale_wordlists,
+        project_wordlist
       ]
     end
   end

data/lib/spellr/line_location.rb

@@ -2,28 +2,34 @@
 
 module Spellr
   class LineLocation
-    attr_reader :file
     attr_reader :line_number
     attr_reader :char_offset
     attr_reader :byte_offset
 
     def initialize(file = '[String]', line_number = 1, char_offset: 0, byte_offset: 0)
-      @file = file
+      @filename = file
       @line_number = line_number
       @char_offset = char_offset
       @byte_offset = byte_offset
     end
 
     def to_s
-      "#{relative_file_name}:#{line_number}"
+      "#{file_relative_path}:#{line_number}"
     end
 
-    def file_name
-      file.respond_to?(:to_path) ? file.to_path : file
+    def file_relative_path
+      file.relative_path
     end
 
-    def relative_file_name
-      Pathname.new(file_name).relative_path_from(Pathname.pwd)
+    def file
+      @file ||= Spellr::File.wrap(@filename)
+    end
+
+    def advance(line)
+      LineLocation.new(@filename,
+                       line_number + 1,
+                       char_offset: char_offset + line.length,
+                       byte_offset: byte_offset + line.bytesize)
     end
   end
 end

data/lib/spellr/line_tokenizer.rb

@@ -5,57 +5,62 @@ require_relative '../spellr'
 require_relative 'column_location'
 require_relative 'token'
 require_relative 'key_tuner/naive_bayes'
+require_relative 'token_regexps'
 
 module Spellr
-  class LineTokenizer < StringScanner # rubocop:disable Metrics/ClassLength
+  class LineTokenizer < StringScanner
     attr_reader :line
     attr_accessor :disabled
     alias_method :disabled?, :disabled
-    attr_accessor :skip_uri
-    alias_method :skip_uri?, :skip_uri
     attr_accessor :skip_key
     alias_method :skip_key?, :skip_key
 
-    def initialize(*line, skip_uri: true, skip_key: true)
-      @line = Spellr::Token.wrap(line.first)
-      @skip_uri = skip_uri
+    include TokenRegexps
+
+    def initialize(line, skip_key: true)
+      @line = line
       @skip_key = skip_key
 
       super(@line.to_s)
     end
 
     def string=(line)
-      @line = Token.wrap(line)
+      @line = line
       super(@line.to_s)
     end
 
     def each_term
       until eos?
         term = next_term
-        next unless term
-        next if disabled?
+        next if !term || disabled?
 
         yield term
       end
     end
 
-    def each_token
+    def each_token(skip_term_proc: nil) # rubocop:disable Metrics/MethodLength
       until eos?
         term = next_term
         next unless term
-        next if disabled?
+        next if disabled? || skip_term_proc&.call(term)
 
         yield Token.new(term, line: line, location: column_location(term))
       end
     end
 
+    # jump to character-aware position
+    # TODO: handle jump backward
+    def charpos=(new_charpos)
+      skip(/.{#{new_charpos - charpos}}/m)
+    end
+
     private
 
     def column_location(term)
       ColumnLocation.new(
         byte_offset: pos - term.bytesize,
         char_offset: charpos - term.length,
-        line_location: line.location.line_location
+        **(line.respond_to?(:location) ? { line_location: line.location.line_location } : {})
       )
     end
 
@@ -64,23 +69,10 @@ module Spellr
     end
 
     def next_term
-      if skip_nonwords_and_flags
-        nil
-      else
-        scan_term
-      end
-    end
+      return if skip_nonwords_and_flags
 
-    # [Word], [Word]Word [Word]'s [Wordn't]
-    TITLE_CASE_RE = /[[:upper:]][[:lower:]]+(?:['’][[:lower:]]+(?<!['’]s))*/.freeze
-    # [WORD] [WORD]Word [WORDN'T] [WORD]'S [WORD]'s [WORD]s
-    UPPER_CASE_RE = /[[:upper:]]+(?:['’][[:upper:]]+(?<!['’][Ss]))*(?:(?![[:lower:]])|(?=s(?![[:lower:]])))/.freeze
-    # [word] [word]'s [wordn't]
-    LOWER_CASE_RE = /[[:lower:]]+(?:['’][[:lower:]]+(?<!['’]s))*/.freeze
-    # for characters in [:alpha:] that aren't in [:lower:] or [:upper:] e.g. Arabic
-    OTHER_CASE_RE = /(?:[[:alpha:]](?<![[:lower:][:upper:]]))+/.freeze
-
-    TERM_RE = Regexp.union(TITLE_CASE_RE, UPPER_CASE_RE, LOWER_CASE_RE, OTHER_CASE_RE)
+      scan_term
+    end
 
     def scan_term
       term = scan(TERM_RE)
@@ -88,126 +80,35 @@ module Spellr
       return term if term && term.length >= Spellr.config.word_minimum_length
     end
 
-    NOT_EVEN_NON_WORDS_RE = %r{[^[:alpha:]/%#0-9\\]+}.freeze # everything not covered by more specific skips/scans
-    LEFTOVER_NON_WORD_BITS_RE = %r{[/%#\\]|\d+}.freeze # e.g. a / not starting //a-url.com
-    HEX_RE = /(?:#(?:\h{6}|\h{3})|0x\h+)(?![[:alpha:]])/.freeze
-    SHELL_COLOR_ESCAPE_RE = /\\(?:e|0?33)\[\d+(;\d+)*m/.freeze
-    PUNYCODE_RE = /xn--[a-v0-9\-]+(?:[[:alpha:]])/.freeze
-    BACKSLASH_ESCAPE_RE = /\\[a-zA-Z]/.freeze # TODO: hex escapes e.g. \xAA. TODO: language aware escapes
-    REPEATED_SINGLE_LETTERS_RE = /(?:([[:alpha:]])\1+)(?![[:alpha:]])/.freeze # e.g. xxxxxxxx (it's not a word)
-    URL_ENCODED_ENTITIES_RE = /%[0-8A-F]{2}/.freeze
-    # There's got to be a better way of writing this
-    SEQUENTIAL_LETTERS_RE = /a(?:b(?:c(?:d(?:e(?:f(?:g(?:h(?:i(?:j(?:k(?:l(?:m(?:n(?:o(?:p(?:q(?:r(?:s(?:t(?:u(?:v(?:w(?:x(?:yz?)?)?)?)?)?)?)?)?)?)?)?)?)?)?)?)?)?)?)?)?)?)?)?)?(?![[:alpha:]])/i.freeze # rubocop:disable Metrics/LineLength
-
-    # I didn't want to do this myself. BUT i need something to heuristically match on, and it's difficult
-    URL_SCHEME = '(//|https?://|s?ftp://|mailto:)'
-    URL_USERINFO = '([[:alnum:]]+(?::[[:alnum:]]+)?@)'
-    URL_HOSTNAME = '((?:[[:alnum:]-]+(?:\\\\?\\.[[:alnum:]-]+)+|localhost|\\d{1,3}(?:\\.\\d{1,3}){3}))'
-    URL_PORT = '(:\\d+)'
-    URL_PATH = '(/(?:[[:alnum:]=@!$&\\-/._\\\\]|%\h{2})+)'
-    URL_QUERY = '(\\?(?:[[:alnum:]=!$\\-/.\\\\]|%\\h{2})+(?:&(?:[[:alnum:]=!$\\-/.\\\\]|%\\h{2})+)*)'
-    URL_FRAGMENT = '(\\#(?:[[:alnum:]=!$&\\-/.\\\\]|%\\h{2})+)'
-    URL_RE = /
-      (?:
-        #{URL_SCHEME}#{URL_USERINFO}?#{URL_HOSTNAME}#{URL_PORT}?#{URL_PATH}?
-        |
-        #{URL_SCHEME}?#{URL_USERINFO}#{URL_HOSTNAME}#{URL_PORT}?#{URL_PATH}?
-        |
-        #{URL_SCHEME}?#{URL_USERINFO}?#{URL_HOSTNAME}#{URL_PORT}?#{URL_PATH}
-      )
-      #{URL_QUERY}?#{URL_FRAGMENT}?
-    /x.freeze
-
-    KNOWN_KEY_PATTERNS_RE = %r{(
-      SG\.[\w\-]{22}\.[\w\-]{43} | # sendgrid
-      prg-\h{8}-\h{4}-\h{4}-\h{4}-\h{12} | # hyperwallet
-      GTM-[A-Z0-9]{7} | # google tag manager
-      sha1-[A-Za-z0-9=+/]{28} |
-      sha512-[A-Za-z0-9=+/]{88} |
-      data:[a-z/;0-9\-]+;base64,[A-Za-z0-9+/]+=*(?![[:alnum:]])
-    )}x.freeze
-
-    SKIPS = Regexp.union(
-      NOT_EVEN_NON_WORDS_RE,
-      SHELL_COLOR_ESCAPE_RE,
-      BACKSLASH_ESCAPE_RE,
-      URL_ENCODED_ENTITIES_RE,
-      HEX_RE,
-      URL_RE, # 2%
-      KNOWN_KEY_PATTERNS_RE
-    ).freeze
-
-    AFTER_KEY_SKIPS = Regexp.union(
-      LEFTOVER_NON_WORD_BITS_RE,
-      REPEATED_SINGLE_LETTERS_RE,
-      SEQUENTIAL_LETTERS_RE
-    )
-
     def skip_nonwords
-      skip(SKIPS) ||
-        skip_key_heuristically || # 5%
-        skip(AFTER_KEY_SKIPS)
+      skip(SKIPS) || skip_key_heuristically || skip(AFTER_KEY_SKIPS)
     end
 
-    KEY_RE = %r{[A-Za-z0-9]([A-Za-z0-9+/\-_]*)=*(?![[:alnum:]])}.freeze
-    N = NaiveBayes.new
-    def skip_key_heuristically # rubocop:disable Metrics/MethodLength
-      return unless scan(KEY_RE)
-      # I've come across some large base64 strings by this point they're definitely base64.
-      return true if matched.length > 200
-
-      if key_roughly?(matched)
-        if N.key?(matched)
-          true
-        else
-          unscan
-          false
-        end
-      else
-        unscan
-        false
-      end
-    end
+    def skip_key_heuristically
+      possible_key = check(POSSIBLE_KEY_RE)
 
-    # this is in a method because the minimum word length stuff was throwing it off
-    # TODO: move to config maybe?
-    def min_alpha_re
-      /(?:
-        [A-Z][a-z]{#{Spellr.config.word_minimum_length - 1}}
-        |
-        [a-z]{#{Spellr.config.word_minimum_length}}
-        |
-        [A-Z]{#{Spellr.config.word_minimum_length}}
-      )/x.freeze
-    end
-    ALPHA_SEP_RE = '[A-Za-z][A-Za-z\\-_/+]*'
-    NUM_SEP_RE = '\\d[\\d\\-_/+]*'
-    THREE_CHUNK_RE = /^(?:
-      #{ALPHA_SEP_RE}#{NUM_SEP_RE}#{ALPHA_SEP_RE}
-      |
-      #{NUM_SEP_RE}#{ALPHA_SEP_RE}#{NUM_SEP_RE}
-    )/x.freeze
-    def key_roughly?(matched)
-      return unless matched.length >= Spellr.config.key_minimum_length
-      return unless matched.match?(THREE_CHUNK_RE)
-      return unless matched.match?(min_alpha_re) # or there's no point
-
-      true
+      return unless possible_key
+      return unless key?(possible_key)
+
+      self.pos += possible_key.bytesize
     end
 
-    # jump to character-aware position
-    def charpos=(new_charpos)
-      skip(/.{#{new_charpos - charpos}}/m)
+    BAYES_KEY_HEURISTIC = NaiveBayes.new
+    def key?(possible_key)
+      # I've come across some large base64 strings by this point they're definitely base64.
+      return true if possible_key.length > 200
+      return unless possible_key.length >= Spellr.config.key_minimum_length
+      return unless possible_key.match?(min_alpha_re) # or there's no point
+
+      BAYES_KEY_HEURISTIC.key?(possible_key)
     end
 
-    SPELLR_DISABLE_RE = /spellr:disable/.freeze
     def skip_and_track_disable
       return if disabled?
 
       skip(SPELLR_DISABLE_RE) && self.disabled = true
     end
 
-    SPELLR_ENABLE_RE = /spellr:enable/.freeze
     def skip_and_track_enable
       return unless disabled?
 

data/lib/spellr/output.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Spellr
+  class Output
+    attr_reader :exit_code
+
+    def initialize
+      @exit_code = 0
+    end
+
+    def stdin
+      @stdin ||= $stdin
+    end
+
+    def stdout
+      @stdout ||= $stdout
+    end
+
+    def stderr
+      @stderr ||= $stderr
+    end
+
+    def stdout?
+      defined?(@stdout)
+    end
+
+    def stderr?
+      defined?(@stderr)
+    end
+
+    def counts
+      @counts ||= Hash.new(0)
+    end
+
+    def exit_code=(value)
+      @exit_code = value unless value.zero?
+    end
+
+    def increment(counter)
+      counts[counter] += 1
+    end
+
+    def puts(str)
+      stdout.puts(str)
+    end
+
+    def warn(str)
+      stderr.puts(str)
+    end
+
+    def print(str)
+      stdout.print(str)
+    end
+
+    def <<(other) # rubocop:disable Metrics/AbcSize
+      self.exit_code = other.exit_code
+      stderr.puts other.stderr.string if other.stderr?
+      stdout.puts other.stdout.string if other.stdout?
+      counts.merge!(other.counts) { |_k, a, b| a + b }
+    end
+  end
+end

data/lib/spellr/output_stubbed.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require_relative 'output'
+
+module Spellr
+  class OutputStubbed < Spellr::Output
+    attr_accessor :exit_code
+
+    def initialize
+      @exit_code = 0
+    end
+
+    def stdin
+      @stdin ||= StringIO.new
+    end
+
+    def stdout
+      @stdout ||= StringIO.new
+    end
+
+    def stderr
+      @stderr ||= StringIO.new
+    end
+
+    def marshal_dump # rubocop:disable Metrics/MethodLength
+      {
+        exit_code: exit_code,
+        counts: @counts,
+        stdin: @stdin&.string,
+        stdin_pos: @stdin&.pos,
+        stdout: @stdout&.string,
+        stdout_pos: @stdout&.pos,
+        stderr: @stderr&.string,
+        stderr_pos: @stderr&.pos
+      }
+    end
+
+    def marshal_load(dumped) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      if dumped[:stdin]
+        @stdin = StringIO.new(dumped[:stdin])
+        @stdin.pos = dumped[:stdin_pos]
+      end
+
+      if dumped[:stdout]
+        @stdout = StringIO.new(dumped[:stdout])
+        @stdout.pos = dumped[:stdout_pos]
+      end
+
+      if dumped[:stderr]
+        @stderr = StringIO.new(dumped[:stderr])
+        @stderr.pos = dumped[:stderr_pos]
+      end
+
+      @exit_code = dumped[:exit_code]
+      @counts = dumped[:counts]
+    end
+  end
+end

data/lib/spellr/quiet_reporter.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require_relative 'base_reporter'
+
+module Spellr
+  class QuietReporter < Spellr::BaseReporter
+    def output
+      @output ||= Spellr::OutputStubbed.new
+    end
+
+    def call(_token); end
+  end
+end

data/lib/spellr/reporter.rb

@@ -1,27 +1,23 @@
 # frozen_string_literal: true
 
-require_relative 'string_format'
+require_relative 'base_reporter'
 
 module Spellr
-  class Reporter
-    include Spellr::StringFormat
-
-    attr_accessor :total
-
-    def initialize
-      @total = 0
+  class Reporter < Spellr::BaseReporter
+    def parallel?
+      true
     end
 
-    def finish(checked)
+    def finish
       puts "\n"
-      puts "#{pluralize 'file', checked} checked"
-      puts "#{pluralize 'error', total} found"
+      puts "#{pluralize 'file', counts[:checked]} checked"
+      puts "#{pluralize 'error', counts[:total]} found"
     end
 
     def call(token)
-      puts "#{aqua token.location} #{token.line.highlight(token.char_range).strip}"
+      super
 
-      self.total += 1
+      increment(:total)
     end
   end
 end

data/lib/spellr/token.rb

@@ -1,18 +1,13 @@
 # frozen_string_literal: true
 
-# frozen string_literal: true
-
 require_relative 'column_location'
 require_relative 'string_format'
 
 class String
-  def normalize
-    normalize_cache[to_s]
-  end
-
-  def normalize_cache
-    @@normalize_cache ||= Hash.new do |cache, term| # rubocop:disable Style/ClassVars # i want this shared with subclasses
-      cache[term] = term.strip.downcase.unicode_normalize.tr('’', "'") + "\n"
+  def spellr_normalize
+    @@spellr_normalize ||= {} # rubocop:disable Style/ClassVars # I want to share this with subclasses
+    @@spellr_normalize.fetch(to_s) do |term|
+      @@spellr_normalize[term] = "#{term.strip.downcase.unicode_normalize.tr('’', "'")}\n"
     end
   end
 end
@@ -49,6 +44,11 @@ module Spellr
       )
     end
 
+    def line=(new_line)
+      @line = new_line
+      location.line_location = new_line.location.line_location
+    end
+
     def inspect
       "#<#{self.class.name} #{to_s.inspect} @#{location}>"
     end
@@ -61,6 +61,10 @@ module Spellr
       @byte_range ||= location.byte_offset...(location.byte_offset + bytesize)
     end
 
+    def file_char_range
+      @file_char_range ||= location.absolute_char_offset...(location.absolute_char_offset + length)
+    end
+
     def coordinates
       location.coordinates
     end
@@ -71,13 +75,7 @@ module Spellr
 
     def replace(replacement)
       @replacement = replacement
-      ::File.open(file_name, 'r+') do |f|
-        body = f.read
-        body[location.absolute_char_offset...(location.absolute_char_offset + length)] = replacement
-        f.rewind
-        f.truncate(0)
-        f.write(body)
-      end
+      location.file.insert(replacement, file_char_range)
     end
 
     def file_name