spellr 0.6.0 → 0.7.0

data/lib/spellr/key_tuner/naive_bayes.rb

@@ -9,8 +9,13 @@ require 'yaml'
 class NaiveBayes
   YAML_PATH = File.join(__dir__, 'data.yml')
 
+  attr_reader :feature_set
+  attr_reader :num_classes
+  attr_reader :classes
+  attr_reader :features
+
   def initialize(path = YAML_PATH)
-    load_from_yaml(path) if File.exist?(path)
+    load_from_yaml(path)
     @key = {}
   end
 
@@ -21,7 +26,7 @@ class NaiveBayes
   end
 
   def load_from_yaml(path = YAML_PATH)
-    data = YAML.safe_load(::File.read(path), [Symbol])
+    data = YAML.safe_load(::File.read(path), permitted_classes: [Symbol])
 
     @feature_set = data[:feature_set]
     @num_classes = data[:num_classes]
@@ -29,63 +34,6 @@ class NaiveBayes
     @features = data[:features]
   end
 
-  def save_to_yaml(path = YAML_PATH)
-    write_yaml(path,
-               feature_set: feature_set,
-               num_classes: num_classes,
-               classes: classes,
-               features: features)
-  end
-
-  private
-
-  def write_yaml(path = YAML_PATH, **hash)
-    require 'yaml'
-
-    File.write(path, hash.to_yaml)
-  end
-
-  def training_data
-    @training_data ||= PossibleKey.keys.each_with_object({}) do |key, data|
-      data[key.classification] ||= []
-      data[key.classification] << key.features
-    end
-  end
-
-  def num_classes
-    @num_classes ||= training_data&.length
-  end
-
-  def classes
-    @classes ||= training_data&.keys
-  end
-
-  def features
-    @features ||= training_data.first.last.first.keys
-  end
-
-  def feature_set
-    @feature_set ||= classes.each.with_object({}) do |class_name, feature_set|
-      feature_set[class_name] = features.each.with_object({}) do |feature, feature_set_for_class|
-        feature_set_for_class[feature] = feature_stats_for_class(class_name, feature)
-      end
-    end
-  end
-
-  def feature_stats_for_class(class_name, feature)
-    values = training_data[class_name].map { |row| row[feature] }
-
-    feature_stats(values)
-  end
-
-  def feature_stats(values)
-    {
-      standard_deviation: Stats.standard_deviation(values),
-      mean: Stats.mean(values),
-      variance: Stats.variance(values)
-    }
-  end
-
   # given a class, this method determines the probability
   # of a certain value occurring for a given feature
   # feature: name of the feature in consideration in the training data

data/lib/spellr/key_tuner/possible_key.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require 'pathname'
 require_relative 'stats'
+require_relative '../backports'
 
 class PossibleKey # rubocop:disable Metrics/ClassLength
   include Stats
@@ -15,29 +15,11 @@ class PossibleKey # rubocop:disable Metrics/ClassLength
     B C D F G H J K L M N P Q R S T V W X Y Z
   }.freeze
   BASE_64 = VOWELS + CONSONANTS + %i{0 1 2 3 4 5 6 7 8 9 - _ + / =}.freeze
-  LETTER_COUNT_HASH = BASE_64.map { |k| [k.to_sym, 0] }.to_h
+  letter_count_hash = BASE_64.map { |k| [k.to_sym, 0] }.to_h
+  letter_count_hash.default = 0
+  LETTER_COUNT_HASH = letter_count_hash
   FEATURE_LETTERS = %i{+ - _ / A z Z q Q X x}.freeze
 
-  class << self
-    def keys
-      @keys ||= begin
-        load_from_file('false_positives.txt', false) +
-          load_from_file('keys.txt', true)
-      end
-    end
-
-    private
-
-    def load_from_file(filename, key)
-      Pathname.new(__dir__).join('data', filename).each_line.map! do |line|
-        line = line.chomp
-        next if line.empty?
-
-        PossibleKey.new(line, key)
-      end.compact
-    end
-  end
-
   attr_reader :string
 
   def initialize(string, key = nil)
@@ -100,8 +82,6 @@ class PossibleKey # rubocop:disable Metrics/ClassLength
     when /^[a-z0-9]+$/ then :lower36
     when /^[A-Z0-9]+$/ then :upper36
     when %r{^[A-Za-z0-9\-_+/]+={0,2}$} then :base64
-    else
-      raise "#{string.inspect} is an unrecognised character set"
     end
   end
 
@@ -111,6 +91,7 @@ class PossibleKey # rubocop:disable Metrics/ClassLength
     when :lower36 then 36
     when :upper36 then 36
     when :base64 then 64
+    else 0
     end
   end
 

data/lib/spellr/key_tuner/stats.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative '../backports'
+
 module Stats
   module_function
 

data/lib/spellr/language.rb

@@ -1,6 +1,9 @@
 # frozen_string_literal: true
 
 require_relative 'wordlist'
+require_relative 'file'
+require 'pathname'
+require 'fast_ignore'
 
 module Spellr
   class Language
@@ -30,7 +33,7 @@ module Spellr
 
     def project_wordlist
       @project_wordlist ||= Spellr::Wordlist.new(
-        Pathname.pwd.join('.spellr_wordlists', "#{name}.txt"),
+        Spellr.config.pwd.join('.spellr_wordlists', "#{name}.txt"),
         name: name
       )
     end
@@ -49,7 +52,9 @@ module Spellr
     def matches_includes?(file)
       return @hashbangs.empty? if @includes.empty?
 
-      @fast_ignore ||= FastIgnore.new(include_rules: @includes, gitignore: false)
+      @fast_ignore ||= FastIgnore.new(
+        include_rules: @includes, gitignore: false, root: Spellr.config.pwd_s
+      )
       @fast_ignore.allowed?(file.to_s)
     end
 
@@ -67,12 +72,6 @@ module Spellr
       end
     end
 
-    def load_wordlists(name, paths)
-      wordlists = paths + default_wordlist_paths(name)
-
-      wordlists.map(&Spellr::Wordlist.method(:new))
-    end
-
     def default_wordlists
       [
         gem_wordlist,

data/lib/spellr/line_location.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative 'file'
+
 module Spellr
   class LineLocation
     attr_reader :line_number
@@ -24,12 +26,5 @@ module Spellr
     def file
       @file ||= Spellr::File.wrap(@filename)
     end
-
-    def advance(line)
-      LineLocation.new(@filename,
-                       line_number + 1,
-                       char_offset: char_offset + line.length,
-                       byte_offset: byte_offset + line.bytesize)
-    end
   end
 end

data/lib/spellr/line_tokenizer.rb

@@ -48,19 +48,12 @@ module Spellr
       end
     end
 
-    # jump to character-aware position
-    # TODO: handle jump backward
-    def charpos=(new_charpos)
-      skip(/.{#{new_charpos - charpos}}/m)
-    end
-
     private
 
     def column_location(term)
       ColumnLocation.new(
         byte_offset: pos - term.bytesize,
-        char_offset: charpos - term.length,
-        **(line.respond_to?(:location) ? { line_location: line.location.line_location } : {})
+        char_offset: charpos - term.length
       )
     end
 
@@ -97,7 +90,6 @@ module Spellr
     def key?(possible_key)
       # I've come across some large base64 strings by this point they're definitely base64.
       return true if possible_key.length > 200
-      return unless possible_key.length >= Spellr.config.key_minimum_length
       return unless possible_key.match?(min_alpha_re) # or there's no point
 
       BAYES_KEY_HEURISTIC.key?(possible_key)

data/lib/spellr/output.rb

@@ -2,10 +2,8 @@
 
 module Spellr
   class Output
-    attr_reader :exit_code
-
-    def initialize
-      @exit_code = 0
+    def exit_code
+      @exit_code ||= 0
     end
 
     def stdin
@@ -52,10 +50,10 @@ module Spellr
       stdout.print(str)
     end
 
-    def <<(other) # rubocop:disable Metrics/AbcSize
+    def <<(other)
       self.exit_code = other.exit_code
-      stderr.puts other.stderr.string if other.stderr?
-      stdout.puts other.stdout.string if other.stdout?
+      warn other.stderr.string if other.stderr?
+      puts other.stdout.string if other.stdout?
       counts.merge!(other.counts) { |_k, a, b| a + b }
     end
   end

data/lib/spellr/output_stubbed.rb

@@ -4,12 +4,6 @@ require_relative 'output'
 
 module Spellr
   class OutputStubbed < Spellr::Output
-    attr_accessor :exit_code
-
-    def initialize
-      @exit_code = 0
-    end
-
     def stdin
       @stdin ||= StringIO.new
     end
@@ -23,15 +17,21 @@ module Spellr
     end
 
     def marshal_dump # rubocop:disable Metrics/MethodLength
+      l_exit_code = @exit_code if defined?(@exit_code)
+      l_counts = @counts if defined?(@counts)
+      l_stdin = @stdin if defined?(@stdin)
+      l_stdout = @stdout if defined?(@stdout)
+      l_stderr = @stderr if defined?(@stderr)
+
       {
-        exit_code: exit_code,
-        counts: @counts,
-        stdin: @stdin&.string,
-        stdin_pos: @stdin&.pos,
-        stdout: @stdout&.string,
-        stdout_pos: @stdout&.pos,
-        stderr: @stderr&.string,
-        stderr_pos: @stderr&.pos
+        exit_code: l_exit_code,
+        counts: l_counts,
+        stdin: l_stdin&.string,
+        stdin_pos: l_stdin&.pos,
+        stdout: l_stdout&.string,
+        stdout_pos: l_stdout&.pos,
+        stderr: l_stderr&.string,
+        stderr_pos: l_stderr&.pos
       }
     end
 
@@ -51,8 +51,8 @@ module Spellr
         @stderr.pos = dumped[:stderr_pos]
       end
 
-      @exit_code = dumped[:exit_code]
-      @counts = dumped[:counts]
+      @exit_code = dumped[:exit_code] if dumped[:exit_code]
+      @counts = dumped[:counts] if dumped[:counts]
     end
   end
 end

data/lib/spellr/quiet_reporter.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'base_reporter'
+require_relative 'output_stubbed'
 
 module Spellr
   class QuietReporter < Spellr::BaseReporter

data/lib/spellr/reporter.rb

@@ -4,10 +4,6 @@ require_relative 'base_reporter'
 
 module Spellr
   class Reporter < Spellr::BaseReporter
-    def parallel?
-      true
-    end
-
     def finish
       puts "\n"
       puts "#{pluralize 'file', counts[:checked]} checked"

data/lib/spellr/string_format.rb

@@ -8,39 +8,32 @@ module Spellr
       "#{count} #{word}#{'s' if count != 1}"
     end
 
-    # TODO: make it work without color
-    def color_enabled?
-      true
-    end
-
     def aqua(text)
-      return text unless Spellr::StringFormat.color_enabled?
-
       "\e[36m#{text}#{normal}"
     end
 
     def normal(text = '')
-      return text unless Spellr::StringFormat.color_enabled?
-
       "\e[0m#{text}"
     end
 
     def bold(text)
-      return text unless Spellr::StringFormat.color_enabled?
-
       "\e[1;39m#{text}#{normal}"
     end
 
-    def red(text)
-      return text unless Spellr::StringFormat.color_enabled?
+    def lighten(text)
+      "\e[2;39m#{text}#{normal}"
+    end
 
+    def red(text)
       "\e[1;31m#{text}#{normal}"
     end
 
     def green(text)
-      return text unless Spellr::StringFormat.color_enabled?
-
       "\e[1;32m#{text}#{normal}"
     end
+
+    def key(label)
+      "[#{bold label[0]}]#{label[1..-1]}"
+    end
   end
 end

data/lib/spellr/token.rb

@@ -17,53 +17,41 @@ module Spellr
   class Token < String
     attr_reader :location, :line, :replacement
 
-    def self.wrap(value)
-      return value if value.is_a?(Spellr::Token)
-
-      Spellr::Token.new(value || '')
-    end
-
     def initialize(string, line: string, location: ColumnLocation.new)
       @location = location
       @line = line
       super(string)
     end
 
-    def strip
-      @strip ||= begin
-        lstripped = lstrip
-        new_column_location = lstripped_column_location(lstripped)
-        Token.new(lstripped.rstrip, line: line, location: new_column_location)
-      end
-    end
-
-    def lstripped_column_location(lstripped)
-      ColumnLocation.new(
-        byte_offset: bytesize - lstripped.bytesize,
-        char_offset: length - lstripped.length,
-        line_location: location.line_location
-      )
-    end
-
     def line=(new_line)
       @line = new_line
       location.line_location = new_line.location.line_location
     end
 
+    # :nocov:
     def inspect
       "#<#{self.class.name} #{to_s.inspect} @#{location}>"
     end
+    # :nocov:
 
     def char_range
-      @char_range ||= location.char_offset...(location.char_offset + length)
+      @char_range ||=
+        location.char_offset...(location.char_offset + length)
     end
 
     def byte_range
-      @byte_range ||= location.byte_offset...(location.byte_offset + bytesize)
+      @byte_range ||=
+        location.byte_offset...(location.byte_offset + bytesize)
     end
 
     def file_char_range
-      @file_char_range ||= location.absolute_char_offset...(location.absolute_char_offset + length)
+      @file_char_range ||=
+        location.absolute_char_offset...(location.absolute_char_offset + length)
+    end
+
+    def file_byte_range
+      @file_byte_range ||=
+        location.absolute_byte_offset...(location.absolute_byte_offset + bytesize)
     end
 
     def coordinates
@@ -78,9 +66,5 @@ module Spellr
       @replacement = replacement
       location.file.insert(replacement, file_char_range)
     end
-
-    def file_name
-      location.file_name
-    end
   end
 end

data/lib/spellr/token_regexps.rb

@@ -9,7 +9,7 @@ module Spellr
     # [Word], [Word]Word [Word]'s [Wordn't]
     TITLE_CASE_RE = /[[:upper:]][[:lower:]]+(?:['’][[:lower:]]+(?<!['’]s))*/.freeze
     # [WORD] [WORD]Word [WORDN'T] [WORD]'S [WORD]'s [WORD]s
-    UPPER_CASE_RE = /[[:upper:]]+(?:['’][[:upper:]]+(?<!['’][Ss]))*(?:(?![[:lower:]])|(?=s(?![[:lower:]])))/.freeze # rubocop:disable Metrics/LineLength
+    UPPER_CASE_RE = /[[:upper:]]+(?:['’][[:upper:]]+(?<!['’][Ss]))*(?:(?![[:lower:]])|(?=s(?![[:lower:]])))/.freeze # rubocop:disable Layout/LineLength
     # [word] [word]'s [wordn't]
     LOWER_CASE_RE = /[[:lower:]]+(?:['’][[:lower:]]+(?<!['’]s))*/.freeze
     # for characters in [:alpha:] that aren't in [:lower:] or [:upper:] e.g. Arabic
@@ -30,7 +30,7 @@ module Spellr
     REPEATED_SINGLE_LETTERS_RE = /(?:([[:alpha:]])\1+)(?![[:alpha:]])/.freeze # e.g. xxxxxxxx
     URL_ENCODED_ENTITIES_RE = /%[0-8A-F]{2}/.freeze
     # There's got to be a better way of writing this
-    SEQUENTIAL_LETTERS_RE = /a(?:b(?:c(?:d(?:e(?:f(?:g(?:h(?:i(?:j(?:k(?:l(?:m(?:n(?:o(?:p(?:q(?:r(?:s(?:t(?:u(?:v(?:w(?:x(?:yz?)?)?)?)?)?)?)?)?)?)?)?)?)?)?)?)?)?)?)?)?)?)?)?)?(?![[:alpha:]])/i.freeze # rubocop:disable Metrics/LineLength
+    SEQUENTIAL_LETTERS_RE = /a(?:b(?:c(?:d(?:e(?:f(?:g(?:h(?:i(?:j(?:k(?:l(?:m(?:n(?:o(?:p(?:q(?:r(?:s(?:t(?:u(?:v(?:w(?:x(?:yz?)?)?)?)?)?)?)?)?)?)?)?)?)?)?)?)?)?)?)?)?)?)?)?)?(?![[:alpha:]])/i.freeze # rubocop:disable Layout/LineLength
 
     # I didn't want to do this myself
     # BUT i need something to heuristically match on, and it's difficult

data/lib/spellr/tokenizer.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require_relative '../spellr'
 require_relative 'token'
 require_relative 'column_location'
 require_relative 'line_location'
@@ -78,15 +77,6 @@ module Spellr
 
     attr_reader :line_tokenizer
 
-    def each_line_token
-      line_location = @start_at.line_location
-
-      file.each_line do |line|
-        yield Token.new(line, location: ColumnLocation.new(line_location: line_location))
-        line_location = line_location.advance(line)
-      end
-    end
-
     def prepare_tokenizer_for_line(line)
       line_tokenizer.string = line
       line_tokenizer.pos = 0

data/lib/spellr/validations.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Spellr
+  module Validations
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+    module ClassMethods
+      def validations
+        @validations ||= []
+      end
+
+      def validate(method)
+        validations << method
+      end
+    end
+
+    def valid?
+      self.class.validations.each do |validation|
+        send(validation)
+      end
+
+      errors.empty?
+    end
+
+    def errors
+      @errors ||= []
+    end
+  end
+end

data/lib/spellr/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Spellr
-  VERSION = '0.6.0'
+  VERSION = '0.7.0'
 end

data/lib/spellr/wordlist.rb

@@ -2,7 +2,7 @@
 
 require 'pathname'
 require_relative '../spellr'
-require_relative 'token'
+require_relative 'token' # for spellr_normalize
 
 module Spellr
   class Wordlist
@@ -12,20 +12,20 @@ module Spellr
 
     def initialize(file, name: file)
       path = @file = file
-      @path = Pathname.pwd.join('.spellr_wordlists').join(path).expand_path
+      @path = Spellr.config.pwd.join('.spellr_wordlists').join(path).expand_path
       @name = name
       @include = {}
     end
 
     def each(&block)
-      raise_unless_exists?
-
-      @path.each_line(&block)
+      words.each(&block)
     end
 
+    # :nocov:
     def inspect
       "#<#{self.class.name}:#{@path}>"
     end
+    # :nocov:
 
     # significantly faster than default Enumerable#include?
     # requires terms to have been sorted
@@ -60,12 +60,6 @@ module Spellr
       clear_cache
     end
 
-    def read
-      raise_unless_exists?
-
-      @path.read
-    end
-
     def exist?
       return @exist if defined?(@exist)
 
@@ -80,6 +74,10 @@ module Spellr
       clear_cache
     end
 
+    def length
+      to_a.length
+    end
+
     private
 
     def insert_sorted(term)
@@ -92,11 +90,5 @@ module Spellr
       @include = {}
       remove_instance_variable(:@exist) if defined?(@exist)
     end
-
-    def raise_unless_exists?
-      return if exist?
-
-      raise Spellr::Wordlist::NotFound, "Wordlist file #{@file} doesn't exist at #{@path}"
-    end
   end
 end

data/lib/spellr/wordlist_reporter.rb

@@ -5,10 +5,6 @@ require_relative 'base_reporter'
 
 module Spellr
   class WordlistReporter < Spellr::BaseReporter
-    def parallel?
-      true
-    end
-
     def finish
       output.puts words.sort.join
     end

data/lib/spellr.rb

@@ -10,6 +10,18 @@ module Spellr
     class NotFound < Spellr::Error; end
   end
 
+  class Config
+    class NotFound < Spellr::Error; end
+    class Invalid < Spellr::Error; end
+  end
+
+  class InvalidByteSequence < ArgumentError
+    RE = /invalid byte sequence/.freeze
+    def self.===(error)
+      error.is_a?(ArgumentError) && error.message.match?(RE)
+    end
+  end
+
   module_function
 
   def config

data/spellr.gemspec

@@ -14,7 +14,9 @@ Gem::Specification.new do |spec|
   spec.homepage = 'http://github.com/robotdana/spellr'
   spec.license = 'MIT'
 
-  spec.files = Dir.glob('{lib,exe,bin/fetch_wordlist,wordlists}/**/{*,.*}') + %w{
+  spec.required_ruby_version = '>= 2.4'
+
+  spec.files = Dir.glob('{lib,exe,wordlists}/**/{*,.*}') + %w{
     CHANGELOG.md
     Gemfile
     LICENSE.txt
@@ -26,15 +28,18 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_development_dependency 'bundler', '~> 2.0'
+  spec.add_development_dependency 'mime-types', '~> 3.3.1'
+  spec.add_development_dependency 'nokogiri'
   spec.add_development_dependency 'pry'
-  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rake', '>= 12.3.3'
   spec.add_development_dependency 'rspec', '~> 3.0'
-  spec.add_development_dependency 'rspec-eventually'
   spec.add_development_dependency 'rubocop'
   spec.add_development_dependency 'rubocop-rspec'
-  spec.add_development_dependency 'terminal-table'
-  spec.add_development_dependency 'tty_string'
-  spec.add_development_dependency 'nokogiri'
-  spec.add_dependency 'fast_ignore', '~> 0.5.2'
+  spec.add_development_dependency 'simplecov', '~> 0.18.5'
+  spec.add_development_dependency 'simplecov-console'
+  spec.add_development_dependency 'tty_string', '>= 0.2.1'
+  spec.add_development_dependency 'webmock', '~> 3.8'
+
+  spec.add_dependency 'fast_ignore', '~> 0.6.0'
   spec.add_dependency 'parallel', '~> 1.0'
 end