spellr 0.5.0 → 0.5.1

data/lib/spellr/interactive.rb

@@ -3,52 +3,62 @@
 require 'io/console'
 require 'readline'
 require_relative '../spellr'
-require_relative 'reporter'
-require_relative 'string_format'
+require_relative 'interactive_add'
+require_relative 'interactive_replacement'
+require_relative 'base_reporter'
 
 module Spellr
-  class Interactive # rubocop:disable Metrics/ClassLength
-    include Spellr::StringFormat
-
-    attr_reader :global_replacements, :global_skips
-    attr_reader :global_insensitive_replacements
-    attr_reader :global_insensitive_skips
-    attr_accessor :total_skipped
-    attr_accessor :total_fixed
-    attr_accessor :total_added
+  class Interactive < BaseReporter
+    def parallel?
+      false
+    end
 
-    def finish(checked) # rubocop:disable Metrics/AbcSize
+    def finish # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
       puts "\n"
-      puts "#{pluralize 'file', checked} checked"
+      puts "#{pluralize 'file', counts[:checked]} checked"
       puts "#{pluralize 'error', total} found"
-      puts "#{pluralize 'error', total_skipped} skipped" if total_skipped.positive?
-      puts "#{pluralize 'error', total_fixed} fixed" if total_fixed.positive?
-      puts "#{pluralize 'word', total_added} added" if total_added.positive?
+      if counts[:total_skipped].positive?
+        puts "#{pluralize 'error', counts[:total_skipped]} skipped"
+      end
+      puts "#{pluralize 'error', counts[:total_fixed]} fixed" if counts[:total_fixed].positive?
+      puts "#{pluralize 'word', counts[:total_added]} added" if counts[:total_added].positive?
     end
 
-    def total
-      total_skipped + total_fixed + total_added
+    def global_replacements
+      @global_replacements ||= begin
+        counts[:global_replacements] = {} unless counts.key?(:global_replacements)
+        counts[:global_replacements]
+      end
     end
 
-    def initialize
-      @global_replacements = {}
-      @global_insensitive_replacements = {}
-      @global_skips = []
-      @global_insensitive_skips = []
-      @total_skipped = 0
-      @total_fixed = 0
-      @total_added = 0
+    def global_skips
+      @global_skips ||= begin
+        counts[:global_skips] = [] unless counts.key?(:global_skips)
+        counts[:global_skips]
+      end
     end
 
     def call(token)
       return if attempt_global_replacement(token)
       return if attempt_global_skip(token)
 
-      Spellr::Reporter.new.call(token)
+      super
 
       prompt(token)
     end
 
+    def stdin_getch
+      choice = output.stdin.getch
+      clear_current_line
+      choice
+    end
+
+    private
+
+    def total
+      counts[:total_skipped] + counts[:total_fixed] + counts[:total_added]
+    end
+
     def prompt(token)
       print bold('[r,R,s,S,a,e,?]')
 
@@ -58,21 +68,18 @@ module Spellr
     end
 
     def attempt_global_skip(token)
-      return unless global_skips.include?(token.to_s) ||
-        global_insensitive_skips.include?(token.normalize)
+      return unless global_skips.include?(token.to_s)
 
       puts "Automatically skipped #{red(token)}"
-      self.total_skipped += 1
+      increment(:total_skipped)
     end
 
-    def attempt_global_replacement(token)
-      global_replacement = global_replacements[token.to_s]
-      global_replacement ||= global_insensitive_replacements[token.normalize]
-      return unless global_replacement
+    def attempt_global_replacement(token, replacement = global_replacements[token.to_s])
+      return unless replacement
 
-      token.replace(global_replacement)
-      self.total_fixed += 1
-      puts "Automatically replaced #{red(token)} with #{green(global_replacement)}"
+      token.replace(replacement)
+      increment(:total_fixed)
+      puts "Automatically replaced #{red(token)} with #{green(replacement)}"
       throw :check_file_from, token
     end
 
@@ -81,28 +88,21 @@ module Spellr
     end
 
     def handle_response(token) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
-      task = STDIN.getch
-      clear_current_line
-
-      case task
+      case stdin_getch
       when "\u0003" # ctrl c
         exit 1
       when 'a'
-        handle_add(token)
+        Spellr::InteractiveAdd.new(token, self)
       when 's', "\u0004" # ctrl d
         handle_skip(token)
       when 'S'
         handle_skip(token) { |skip_token| global_skips << skip_token.to_s }
-      when 'i'
-        handle_skip(token) { |skip_token| global_insensitive_skips << skip_token.downcase }
       when 'R'
-        handle_replacement(token) { |replacement| global_replacements[token.to_s] = replacement }
-      when 'I'
-        handle_replacement(token) { |replacement| global_insensitive_replacements[token.normalize] = replacement }
+        Spellr::InteractiveReplacement.new(token, self).global_replace
       when 'r'
-        handle_replacement(token)
+        Spellr::InteractiveReplacement.new(token, self).replace
       when 'e'
-        handle_replace_line(token)
+        Spellr::InteractiveReplacement.new(token, self).replace_line
       when '?'
         handle_help(token)
       else
@@ -112,66 +112,12 @@ module Spellr
     end
 
     def handle_skip(token)
-      self.total_skipped += 1
+      increment(:total_skipped)
       yield token if block_given?
       puts "Skipped #{red(token)}"
     end
 
-    # TODO: handle more than 16 options
-    def handle_add(token) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-      puts "Add #{red(token)} to wordlist:"
-      languages = Spellr.config.languages_for(token.location.file.path)
-
-      languages.each do |language|
-        puts "[#{language.key}] #{language.name}"
-      end
-      choice = STDIN.getch
-      clear_current_line
-      case choice
-      when "\u0003" # ctrl c
-        puts '^C again to exit'
-        call(token)
-      when *languages.map(&:key)
-        wl = languages.find { |w| w.key == choice }.project_wordlist
-
-        wl.add(token)
-        self.total_added += 1
-        puts "Added #{red(token)} to #{wl.name} wordlist"
-        throw :check_file_from, token
-      else
-        handle_add(token)
-      end
-    end
-
-    def handle_replacement(token, original_token: token) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
-      readline_editable_print(token.chomp)
-      highlighted_token = token == original_token ? red(token) : token.highlight(original_token.char_range)
-      puts "#{aqua '>>'} #{highlighted_token.chomp}"
-      prompt = "#{aqua '=>'} "
-      replacement = Readline.readline(prompt)
-      if replacement.empty?
-        call(token)
-      else
-        full_replacement = token == original_token ? replacement : replacement + "\n"
-        token.replace(full_replacement)
-        yield replacement if block_given?
-        self.total_fixed += 1
-        puts "Replaced #{red(token.chomp)} with #{green(replacement.chomp)}"
-        throw :check_file_from, token
-      end
-    rescue Interrupt
-      puts '^C again to exit'
-      call(original_token)
-    end
-
-    def handle_replace_line(token)
-      handle_replacement(
-        token.line,
-        original_token: token
-      )
-    end
-
-    def handle_help(token) # rubocop:disable Metrics/AbcSize
+    def handle_help(token) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
       puts "#{bold '[r]'} Replace #{red token}"
       puts "#{bold '[R]'} Replace all future instances of #{red token}"
       puts "#{bold '[s]'} Skip #{red token}"
@@ -181,16 +127,5 @@ module Spellr
       puts "#{bold '[?]'} Show this help"
       handle_response(token)
     end
-
-    def readline_editable_print(string)
-      Readline.pre_input_hook = lambda {
-        Readline.refresh_line
-        Readline.insert_text string.to_s
-        Readline.redisplay
-
-        # Remove the hook right away.
-        Readline.pre_input_hook = nil
-      }
-    end
   end
 end

data/lib/spellr/interactive_add.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require_relative 'string_format'
+
+module Spellr
+  class InteractiveAdd
+    include Spellr::StringFormat
+
+    attr_reader :token, :reporter
+
+    def initialize(token, reporter)
+      @token = token
+      @reporter = reporter
+
+      ask_wordlist
+    end
+
+    def languages
+      @languages ||= Spellr.config.languages_for(token.location.file.to_path)
+    end
+
+    def language_keys
+      @language_keys ||= @languages.map(&:key)
+    end
+
+    def ask_wordlist
+      puts "Add #{red(token)} to wordlist:"
+
+      languages.each do |language|
+        puts "[#{language.key}] #{language.name}"
+      end
+
+      handle_wordlist_choice(reporter.stdin_getch)
+    end
+
+    def handle_ctrl_c
+      puts '^C again to exit'
+      reporter.call(token)
+    end
+
+    def handle_wordlist_choice(choice) # rubocop:disable Metrics/MethodLength
+      case choice
+      when "\u0003"
+        handle_ctrl_c
+      when *language_keys
+        add_to_wordlist(choice)
+      else
+        ask_wordlist
+      end
+    end
+
+    def add_to_wordlist(choice)
+      wordlist = languages.find { |w| w.key == choice }.project_wordlist
+      wordlist << token
+      reporter.increment(:total_added)
+      puts "Added #{red(token)} to #{wordlist.name} wordlist"
+      throw :check_file_from, token
+    end
+
+    def puts(str)
+      reporter.puts(str)
+    end
+  end
+end

data/lib/spellr/interactive_replacement.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Spellr
+  class InteractiveReplacement
+    include Spellr::StringFormat
+
+    attr_reader :token, :reporter, :original_token, :token_highlight, :suffix
+
+    def initialize(token, reporter)
+      @original_token = @token = token
+      @token_highlight = red(token)
+      @reporter = reporter
+      Readline.input = reporter.output.stdin
+      Readline.output = reporter.output.stdout
+    end
+
+    def global_replace
+      replace { |replacement| reporter.global_replacements[token.to_s] = replacement }
+    end
+
+    def replace_line
+      @token = token.line
+      @token_highlight = token.highlight(original_token.char_range).chomp
+      @suffix = "\n"
+
+      replace
+    end
+
+    def complete_replacement(replacement)
+      token.replace("#{replacement}#{suffix}")
+
+      reporter.increment(:total_fixed)
+      puts "Replaced #{red(token.chomp)} with #{green(replacement)}"
+      throw :check_file_from, token
+    end
+
+    def replace # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      readline_editable_print(token.chomp)
+
+      puts "#{aqua '>>'} #{token_highlight}"
+      replacement = Readline.readline("#{aqua '=>'} ")
+
+      return reporter.call(token) if replacement.empty?
+
+      yield replacement if block_given?
+      complete_replacement(replacement)
+    rescue Interrupt
+      puts '^C again to exit'
+      reporter.call(original_token)
+    end
+
+    private
+
+    def readline_editable_print(string) # rubocop:disable Metrics/MethodLength
+      Readline.pre_input_hook = lambda {
+        Readline.refresh_line
+        Readline.insert_text string.to_s
+        Readline.redisplay
+
+        # Remove the hook right away.
+        Readline.pre_input_hook = nil
+      }
+    end
+
+    def puts(str)
+      reporter.puts(str)
+    end
+  end
+end

data/lib/spellr/key_tuner/naive_bayes.rb

@@ -6,26 +6,22 @@ require 'yaml'
 # this is lifted in whole from this article. i don't understand the maths and i don't want to
 # https://www.sitepoint.com/machine-learning-ruby-naive-bayes-theorem/
 
-class NaiveBayes # rubocop:disable Metrics/ClassLength
-  include Stats
-
+class NaiveBayes
   YAML_PATH = File.join(__dir__, 'data.yml')
 
-  def training_data
-    @training_data ||= begin
-      PossibleKey.load
-      PossibleKey.keys.each.with_object({}) do |key, data|
-        key_class = key.key? ? 'key' : 'not_key'
-        character_set = key.character_set
-        key_key = "#{key_class}_#{character_set}"
-        data[key_key] ||= []
-        data[key_key] << key.features
-      end
+  def initialize(path = YAML_PATH)
+    load_from_yaml(path) if File.exist?(path)
+  end
+
+  def key?(string)
+    @key ||= {}
+    @key.fetch(string) do
+      @key[string] = classify(PossibleKey.new(string).features).start_with?('key')
     end
   end
 
-  def load_from_yaml
-    data = YAML.safe_load(::File.read(YAML_PATH), [Symbol])
+  def load_from_yaml(path = YAML_PATH)
+    data = YAML.safe_load(::File.read(path), [Symbol])
 
     @feature_set = data[:feature_set]
     @num_classes = data[:num_classes]
@@ -33,18 +29,27 @@ class NaiveBayes # rubocop:disable Metrics/ClassLength
     @features = data[:features]
   end
 
-  def save_to_yaml
+  def save_to_yaml(path = YAML_PATH)
+    write_yaml(path,
+               feature_set: feature_set,
+               num_classes: num_classes,
+               classes: classes,
+               features: features)
+  end
+
+  private
+
+  def write_yaml(path = YAML_PATH, **hash)
     require 'yaml'
-    File.write(YAML_PATH, {
-      feature_set: feature_set,
-      num_classes: num_classes,
-      classes: classes,
-      features: features
-    }.to_yaml)
+
+    File.write(path, hash.to_yaml)
   end
 
-  def initialize
-    load_from_yaml if File.exist?(YAML_PATH)
+  def training_data
+    @training_data ||= PossibleKey.keys.each_with_object({}) do |key, data|
+      data[key.classification] ||= []
+      data[key.classification] << key.features
+    end
   end
 
   def num_classes
@@ -59,51 +64,35 @@ class NaiveBayes # rubocop:disable Metrics/ClassLength
     @features ||= training_data.first.last.first.keys
   end
 
-  def feature_set # rubocop:disable Metrics/MethodLength
+  def feature_set
     @feature_set ||= classes.each.with_object({}) do |class_name, feature_set|
-      feature_set[class_name] = {}
-
-      features.each do |feature|
-        values = training_data[class_name].map do |row|
-          row[feature]
-        end
-
-        feature_set[class_name][feature] = {
-          standard_deviation: standard_deviation(values),
-          mean: mean(values),
-          variance: variance(values)
-        }
+      feature_set[class_name] = features.each.with_object({}) do |feature, feature_set_for_class|
+        feature_set_for_class[feature] = feature_stats_for_class(class_name, feature)
       end
     end
   end
 
+  def feature_stats_for_class(class_name, feature)
+    values = training_data[class_name].map { |row| row[feature] }
+
+    feature_stats(values)
+  end
+
+  def feature_stats(values)
+    {
+      standard_deviation: Stats.standard_deviation(values),
+      mean: Stats.mean(values),
+      variance: Stats.variance(values)
+    }
+  end
+
   # given a class, this method determines the probability
   # of a certain value occurring for a given feature
-  # index: index of the feature in consideration in the training data
+  # feature: name of the feature in consideration in the training data
   # value: the value of the feature for which we are finding the probability
   # class_name: name of the class in consideration
-  def feature_probability(feature, value, class_name) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-    # get the feature value set
-    fs = feature_set[class_name][feature]
-
-    # statistical properties of the feature set
-    fs_std = fs[:standard_deviation]
-    fs_mean = fs[:mean]
-    fs_var = fs[:variance]
-
-    # deal with the edge case of a 0 standard deviation
-    if fs_std == 0
-      return fs_mean == value ? 1.0 : 0.0
-    end
-
-    # calculate the gaussian probability
-    pi = Math::PI
-    e = Math::E
-
-    exp = -((value - fs_mean)**2) / (2 * fs_var)
-    probability = (1.0 / Math.sqrt(2 * pi * fs_var)) * (e**exp)
-
-    probability
+  def feature_probability(feature, value, class_name)
+    Stats.gaussian_probability(value, feature_set[class_name][feature])
   end
 
   # multiply together the feature probabilities for all of the
@@ -114,24 +103,6 @@ class NaiveBayes # rubocop:disable Metrics/ClassLength
     end
   end
 
-  def debug(string) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-    require 'terminal-table'
-
-    features = PossibleKey.new(string).features
-
-    table = Terminal::Table.new do |t|
-      t << ['classes'] + classes
-      t << :separator
-      t << ['probabilities'] + classes.map { |c| class_probability(features, c) }
-      features.each do |key, value|
-        t << [key] + classes.map { |c| feature_probability(key, value, c).round(4) }
-      end
-    end
-    puts table
-
-    nil
-  end
-
   # this is where we compute the final naive Bayesian probability
   # for a given set of features being a part of a given class.
   def class_probability(features, class_name)
@@ -141,22 +112,9 @@ class NaiveBayes # rubocop:disable Metrics/ClassLength
     feature_bayes * class_fraction
   end
 
-  # This the method we should be calling!
-  # Given a set of feature values, it decides
-  # what class to categorize them under
   def classify(features)
     classes.max_by do |class_name|
       class_probability(features, class_name)
     end
   end
-
-  def key?(string)
-    key_cache[string]
-  end
-
-  def key_cache
-    @key_cache ||= Hash.new do |cache, string|
-      cache[string] = classify(PossibleKey.new(string).features).start_with?('key')
-    end
-  end
 end

data/lib/spellr/key_tuner/possible_key.rb

@@ -6,23 +6,35 @@ require_relative 'stats'
 class PossibleKey # rubocop:disable Metrics/ClassLength
   include Stats
 
-  class << self
-    attr_reader :keys
-  end
-
-  def self.load # rubocop:disable Metrics/AbcSize
-    @keys = []
+  VOWELS = %i{
+    a e i o u
+    A E I O U
+  }.freeze
+  CONSONANTS = %i{
+    b c d f g h j k l m n p q r s t v w x y z
+    B C D F G H J K L M N P Q R S T V W X Y Z
+  }.freeze
+  BASE_64 = VOWELS + CONSONANTS + %i{0 1 2 3 4 5 6 7 8 9 - _ + / =}.freeze
+  LETTER_COUNT_HASH = BASE_64.map { |k| [k.to_sym, 0] }.to_h
+  FEATURE_LETTERS = %i{+ - _ / A z Z q Q X x}.freeze
 
-    Pathname.new(__dir__).join('data', 'false_positives.txt').each_line do |line|
-      next if line.chomp.empty?
-
-      keys << PossibleKey.new(line.chomp, false)
+  class << self
+    def keys
+      @keys ||= begin
+        load_from_file('false_positives.txt', false) +
+          load_from_file('keys.txt', true)
+      end
     end
 
-    Pathname.new(__dir__).join('data', 'keys.txt').each_line do |line|
-      next if line.chomp.empty?
+    private
 
-      keys << PossibleKey.new(line.chomp, true)
+    def load_from_file(filename, key)
+      Pathname.new(__dir__).join('data', filename).each_line.map! do |line|
+        line = line.chomp
+        next if line.empty?
+
+        PossibleKey.new(line, key)
+      end.compact
     end
   end
 
@@ -35,7 +47,7 @@ class PossibleKey # rubocop:disable Metrics/ClassLength
 
   def features # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
     {
-      **significant_letter_frequency_difference,
+      **letter_frequency_difference_features,
       equal: letter_count[:'='],
       length: length,
       hex: character_set == :hex ? 1 : 0,
@@ -69,24 +81,20 @@ class PossibleKey # rubocop:disable Metrics/ClassLength
     @key
   end
 
+  def classification
+    key_class = key? ? 'key' : 'not_key'
+    "#{key_class}_#{character_set}"
+  end
+
   def length
     string.length
   end
 
-  SIGNIFICANT_LETTERS = %i{+ - _ / A z Z q Q X x}.freeze
-  if RUBY_VERSION >= '2.5'
-    def significant_letter_frequency_difference
-      letter_frequency_difference.slice(*SIGNIFICANT_LETTERS)
-    end
-  else
-    def significant_letter_frequency_difference
-      letter_frequency_difference.each.with_object({}) do |key, value, hash|
-        hash[key] = value if SIGNIFICANT_LETTERS.include?(key)
-      end
-    end
+  def letter_frequency_difference_features
+    letter_frequency_difference.slice(*FEATURE_LETTERS)
   end
 
-  def character_set
+  def character_set # rubocop:disable Metrics/MethodLength
     @character_set ||= case string
     when /^[a-fA-F0-9\-]+$/ then :hex
     when /^[a-z0-9]+$/ then :lower36
@@ -97,7 +105,7 @@ class PossibleKey # rubocop:disable Metrics/ClassLength
     end
   end
 
-  def character_set_total
+  def character_set_total # rubocop:disable Metrics/MethodLength
     case character_set
     when :hex then 16
     when :lower36 then 36
@@ -110,11 +118,9 @@ class PossibleKey # rubocop:disable Metrics/ClassLength
     1.0 / character_set_total * length
   end
 
-  LETTER_COUNT_HASH = (('A'..'Z').to_a + ('a'..'z').to_a + ('0'..'9').to_a + %w{+ _ / - =})
-    .map { |k| [k.to_sym, 0] }.to_h
   def letter_count
     @letter_count ||= begin
-      string.chars.each.with_object(LETTER_COUNT_HASH.dup) do |letter, hash|
+      string.chars.each_with_object(LETTER_COUNT_HASH.dup) do |letter, hash|
         hash[letter.to_sym] += 1
       end
     end
@@ -136,8 +142,6 @@ class PossibleKey # rubocop:disable Metrics/ClassLength
     end
   end
 
-  VOWELS = %i{a e i o u A E I O U}.freeze
-  CONSONANTS = %i{b c d f g h j k l m n p q r s t v w x y z B C D F G H J K L M N P Q R S T V W X Y Z}.freeze
   def vowel_consonant_ratio
     vowels = letter_count.fetch_values(*VOWELS).sum
     consonants = letter_count.fetch_values(*CONSONANTS).sum