forspell 0.0.6 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4dd2b546e45d6a2fab1fb824f628cc2f17d802ce
-  data.tar.gz: e575a2f71a0968b33e715a153d535400dec007e6
+  metadata.gz: 67422b4f2942f8a061b0551a4fa837c0c0c93140
+  data.tar.gz: b56c34d10ecd2bee9dbfa14757ec7853d8a6fe17
 SHA512:
-  metadata.gz: f8f3d245078e74f33da758e09f724d2c911655ee7e4d792e599bd11d7a073fa8ae2acf8972868552939b91a6567d703d3c8972a11adce97bd573fb3c2496a641
-  data.tar.gz: 62826962163488488a5ce89fb72a608fd02a858fbec5e428ec96f8be4980208bc1349eca19d0800967241208b2b2cf833da6536591551f196c62f2df0d6b2885
+  metadata.gz: 1e2512ac8231b29ac095fdf2d71b426a82ba9365c6ae4d9cbf52f349f76b12bf7cdfe170b2001b4f0fe7b3fc4239175f59d8cec7f9742880c30d7a0bc8c88cff
+  data.tar.gz: 432eb94fcf818aba269b8292988956ba905964426e239ae35a9c3972813d6d1b7c19d9a1da2e4146d3f649bfbe13902359ba49fe532458e330e04c6cc4ad89b7

data/exe/clone_repos.sh

@@ -43,7 +43,7 @@ array=( "https://github.com/rspec/rspec-core.git"
         "https://github.com/paper-trail-gem/paper_trail.git"
         "https://github.com/aasm/aasm.git"
         "https://github.com/rubysherpas/paranoia.git"
-        ""
+        "https://github.com/rubocop-hq/rubocop.git"
         ""
         ""
         ""

data/exe/create_dictionary

@@ -1,12 +1,12 @@
 #!/usr/bin/env ruby
 require 'yaml'
-require 'pry'
 require 'csv'
+require 'pry'
 
 class DictionaryCreator
   attr_reader :result
 
-  def initialize(path: '../test/popular_gems/json/')
+  def initialize(path: 'test/popular_gems/')
     @path = path
     @data = {}
     @result = {}
@@ -14,21 +14,20 @@ class DictionaryCreator
 
   def process
     (Dir.entries(@path) - ['.', '..']).each do |filename|
-      # binding.pry
-      @data[filename] = YAML.load_file("#{@path}#{filename}")
+      @data[filename] = File.read("#{@path}#{filename}")
     end
 
     @data.each_pair do |log_filename, data|
       next if data == false
 
       gem_name = log_filename.split('.').first
-      data.each do |data_entry|
-        data_entry[:errors].each do |error|
-          if @result[error]
-            @result[error] = @result[error].merge(count: @result.dig(error, :count) + 1, gems: (@result.dig(error, :gems) + [gem_name]).uniq)
-          else
-            @result[error] = { count: 1, gems: [gem_name] }
-          end
+      data.split(/\n/).each do |error_with_count|
+        error, count = error_with_count.split(' # ')
+        count = count.to_i
+        if @result[error]
+          @result[error] = @result[error].merge(count: @result.dig(error, :count) + count, gems: (@result.dig(error, :gems) + [gem_name]).uniq)
+        else
+          @result[error] = { count: count, gems: [gem_name] }
         end
       end
     end
@@ -37,11 +36,10 @@ class DictionaryCreator
   end
 end
 
-# DictionaryCreator.new.process.result
-
-CSV.open('../ruby_dict.csv', 'wb') do |csv|
+CSV.open('ruby_dict_sort.csv', 'wb') do |csv|
   csv << ['word', 'count', 'gems', 'should_include?']
-  DictionaryCreator.new.process.result.each_pair do |word, data|
-    csv << [word, data[:count], data[:gems].join(', '), nil]
+  res = DictionaryCreator.new.process.result
+  res.keys.sort.each do |word|
+    csv << [word, res[word][:count], res[word][:gems].join(', '), nil]
   end
 end

data/exe/generate_logs

@@ -1,8 +1,11 @@
 #!/bin/bash
 cd ~/forspell
-rm ./test/popular_gems/json/*.log
+rm ./test/popular_gems/*.dict
 
 for var in $(ls ../gem_src/)
 do
-  ./bin/forspell ../gem_src/$var test/popular_gems/json/$var.log
+  cd ~/gem_src/$var
+  rm forspell.dict
+  ~/forspell/exe/forspell --gen-dictionary .
+  cp forspell.dict ~/forspell/test/popular_gems/$var.forspell.dict
 done

data/lib/forspell/cli.rb

@@ -20,9 +20,12 @@ module Forspell
       o.string '-d', '--dictionary-path', 'Path to main hunspell dictionary to use (by default, forspell\'s en_US)', default: 'en_US'
       o.array '-c', '--custom-paths', 'Paths to custom dictionaries', default: []
       o.string '-f', '--format', 'Output formats: readable(default), JSON, YAML', default: 'readable'
-      o.boolean '--gen-dictionary', 'Generate custom dictionary'
+      o.bool '--gen-dictionary', 'Generate custom dictionary', default: false
+      o.bool '--print-filepaths', 'Enable file paths in dictionary mode', default: false
       o.string '-l', '--logfile', 'Log to specified path'
-      o.bool '-v', '--verbose', 'Verbose mode'
+      o.bool '-v', '--verbose', 'Verbose mode', default: false
+      o.bool '--no-suggest', 'Output without suggestions', default: false
+      o.integer '--suggestions-size', 'How many suggestions for each error should be returned', default: 3
       o.on '--help' do
         puts o
         exit
@@ -71,12 +74,14 @@ module Forspell
       end
 
       @opts[:custom_paths] << DEFAULT_CUSTOM_DICT if File.exist?(DEFAULT_CUSTOM_DICT)
+      suggestions_size = (@opts[:gen_dictionary] || @opts[:no_suggest]) ? 0 : @opts[:suggestions_size]
+      suggestions_size ||= 0
 
-      @speller = Speller.new(@opts[:dictionary_path], *@opts[:custom_paths])
+      @speller = Speller.new(@opts[:dictionary_path], *@opts[:custom_paths], suggestions_size: suggestions_size)
     end
 
     def init_reporter
-      @reporter = Reporter.new(**@opts.to_hash.slice(:logfile, :format, :verbose))
+      @reporter = Reporter.new(**@opts.to_hash.slice(:logfile, :format, :verbose, :print_filepaths))
     end
 
     def run

data/lib/forspell/file_list.rb

@@ -16,17 +16,21 @@ module Forspell
     def initialize(paths:, exclude_paths:)
       @paths = paths
       @exclude_paths = exclude_paths
-    end
 
-    def each(&block)
       to_process = @paths.flat_map(&method(:expand_paths))
-
       to_exclude = @exclude_paths.flat_map(&method(:expand_paths))
+      @files = to_process - to_exclude
+    end
 
-      (to_process - to_exclude).map{ |path| path.gsub('//', '/')}
+    def each(&block)
+      @files.map{ |path| path.gsub('//', '/')}
         .each(&block)
     end
 
+    def size
+      @size ||= @files.size
+    end
+
     private
 
     def expand_paths(path)

data/lib/forspell/loaders/markdown.rb

@@ -45,8 +45,10 @@ module Forspell::Loaders
 
       group_by_location = chunks.group_by { |res| res[:location] }
                                 .transform_values do |lines|
-        lines.map { |v| SPECIAL_CHARS_MAP[v[:value]] || v[:value] }.join.split(/\s|,|;|—/)
+        lines.map { |v| SPECIAL_CHARS_MAP[v[:value]] || v[:value] }
+          .join.split(%r{[[:punct:]]&&[^-'_./\\:]|\s})
       end
+      
       group_by_location.each_pair do |location, words|
         words.reject(&:empty?)
              .each { |word| result << Word.new(@file, location || 0, word) }

data/lib/forspell/loaders/ruby.rb

@@ -2,20 +2,30 @@
 
 require 'yard'
 require 'yard/parser/ruby/ruby_parser'
+require 'rdoc'
 require_relative 'source'
 
 module Forspell::Loaders
   class Ruby < Source
+    MAX_COMMENT_LENGTH = 777
+    
+    def initialize(file: nil, text: nil)
+      super
+      @markup = RDoc::Markup.new
+      @formatter = RDoc::Markup::ToMarkdown.new
+      @formatter.width = MAX_COMMENT_LENGTH
+    end
+
     private
 
     def comments
       YARD::Parser::Ruby::RubyParser.new(@input, @file).parse
-        .tokens.select{ |token| token.first == :comment }
-      # example: [:comment, "# def loader_class path\n", [85, 2356]]
+        .tokens.select{ |type,| type == :comment }
+        .reject{ |_, text,| text.start_with?('#  ') }
     end
 
     def text(comment)
-      comment[1]
+      @markup.convert(comment[1], @formatter)
     end
 
     def line(comment)

data/lib/forspell/reporter.rb

@@ -4,18 +4,26 @@ require 'fileutils'
 require 'pastel'
 require 'logger'
 require 'json'
+require 'highline'
+require 'ruby-progressbar'
 
 module Forspell
   class Reporter
     SUCCESS_CODE = 0
     ERROR_CODE = 1
-    ERROR_FORMAT = '%<file>s:%<line>i: %<text>s (suggestions: %<suggestions>s)'
+    DICT_PATH = File.join(Dir.pwd, 'forspell.dict')
+    DICT_OVERWRITE = 'Do you want to overwrite forspell.dict? (yN)'
+    SUGGEST_FORMAT = '(suggestions: %<suggestions>s)'
+    ERROR_FORMAT = '%<file>s:%<line>i: %<text>s %<suggest>s'
     SUMMARY = "Forspell inspects *.rb, *.c, *.cpp, *.md files\n"\
               '%<files>i inspected, %<errors>s detected'
 
+    attr_accessor :progress_bar
+
     def initialize(logfile:,
                    verbose:,
-                   format:)
+                   format:,
+                   print_filepaths: false)
 
       FileUtils.touch(logfile) if logfile.is_a?(String)
       @logger = Logger.new(logfile || STDERR)
@@ -26,6 +34,7 @@ module Forspell
       @pastel = Pastel.new(enabled: $stdout.tty?)
       @errors = []
       @files = []
+      @print_filepaths = print_filepaths
     end
 
     def file(path)
@@ -35,7 +44,7 @@ module Forspell
 
     def error(word, suggestions)
       @errors << [word, suggestions]
-      puts readable(word, suggestions) if @format == 'readable'
+      print(readable(word, suggestions)) if @format == 'readable'
     end
 
     def parsing_error(error)
@@ -64,17 +73,19 @@ module Forspell
     private
 
     def readable(word, suggestions)
+      suggest = format(SUGGEST_FORMAT, suggestions: suggestions.join(', ')) unless suggestions.empty?
+
       format(ERROR_FORMAT,
              file: word[:file],
              line: word[:line],
              text: @pastel.red(word[:text]),
-             suggestions: suggestions.join(', '))
+             suggest: suggest)
     end
 
     def print_formatted
       @errors.map { |word, suggestions| word.to_h.merge(suggestions: suggestions) }
              .public_send("to_#{@format}")
-             .tap { |res| puts res }
+             .tap { |res| print res }
     end
 
     def print_summary
@@ -82,18 +93,32 @@ module Forspell
       color = err_count.positive? ? :red : :green
       total_errors_colorized = @pastel.decorate(err_count.to_s, color)
 
-      puts format(SUMMARY, files: @files.size, errors: total_errors_colorized)
+      print format(SUMMARY, files: @files.size, errors: total_errors_colorized)
     end
 
     def print_dictionary
+      puts DICT_PATH
+      if File.exist?(DICT_PATH)
+        cli = HighLine.new
+        answer = cli.ask(DICT_OVERWRITE)
+        out = answer.downcase == 'y' ? File.new(DICT_PATH, 'w') : exit(1)
+      else
+        out = File.new(DICT_PATH, 'w')
+      end
       @errors.map(&:first)
              .group_by(&:text)
              .transform_values { |v| v.map(&:file).uniq }
              .sort_by { |word, *| word.downcase }
              .each do |text, files|
-        files.each { |file| puts "\# #{file}" }
-        puts @pastel.decorate(text, :red)
+        files.each { |file| out.puts "\# #{file}" } if @print_filepaths
+        out.puts out.tty? ? @pastel.decorate(text, :red) : text
       end
     end
+
+    private
+
+    def print something
+      $stdout.tty? ? @progress_bar&.log(something) : puts(something)
+    end
   end
 end

data/lib/forspell/ruby.dict

@@ -1,77 +1,454 @@
+Cianfrocca
 Gemfile: example
+Hansson
+Heinemeier
 Rakefile
+aasm
 accessor: example
+activerecord
+activesupport
+addon
 admin: example
-args
+amongst
+arel
+arg: argument
+arity
+asciidoc
+asciidoctor
 async
 attr: example
+auth
+authenticatable
+autocorrect
+autocorrected
+autogenerated
+autoload: load
+autoloadable
+autoloaded
+autoloading
+autosave
 backend: example
 backport: port
+backtick
 backtrace: example
+basecamp
+basename: name
+baseurl
+bcrypt
+behaviour: behavior
+benchmarking
+bigint
+bignum
+bing
+binstubs
 bitwise
 boolean: example
+broadcastings
+bugfix: fix
+bugfixes
 builtin
-bundler
+bundler: example
+byebug
+bytesize: size
+capybara: example
+chainable
+changelog: log
+changeset: set
 charset: example
+checkbox: box
+checksum: gum
+chmod
+cli
+cmd
+codebase: base
 codepoint: example
+coderay
+colour: color
+commandline: line
+compat
+compatability
 composable
-config
+concat
+config: example
+confirmable
+const
+constantize
+crossref
+cryptographically
+ctx
+cukes
+customizable
+customizations
+datagram
 dataset: set
+datetime
+declaratively
+decrypt
+dedupe
+deferrable
+defs
+delegator
+deprecations
+deprecator
+desc
+deserialised
 deserialization
 deserialize: rise
-dir
+deserialized
+dest
+destructor
+destructuring
+deterministically
+dev
+diff
+diffable
+dir: beer
+docstring
+docstrings
+dogfoods
+downcase
+downcased
+dup
+dynamoid
+eg
+el
+elsif
 encoding: example
+endian
+enqueue
+enqueued
+enqueues
+enqueuing
 enum
+enumerables
+enums
+env
+epoll
+erb
+errback
+erubi
+erubis
+erubis
+etag
+etag
+eval
+eval'd
+evented
+extensibility
+extname
 fallback: example
+falsey
+falsy
+favour
+fieldset
 filesystem: system
+fixnum
 formatter: example
+freenode
+frontend
+gemspec: specification
+geocode: code
+geocoded
+geocoder
+geocoding
 geospatial
+getter
+gettext
+gitter
+globbing
+gmail
+grapheme
+graphviz
+gsub
+gzip
+gzipped
+habtm
+haml
+hardcode
+hardcoded
+heredoc: document
+highline
+hiredis
+hostname: name
+href
+hstore
+https
 i18n
+iconv
+iframe
+impl
+incrementing
+inflector
+initialised
 initializer: example
 inline
+instantiation
+interdependencies
+invoker
 io
+irb
+iteratively
+ivar: variable
+javascripts
+jbuilder
+jruby
+jruby
+js
+json
+klass
+kqueue
+kramdown
+lcs
 lexer: example
 lib: rib
+libxml
+libxml-ruby
+libxslt
 lifecycle: example
+localhost
+lookups
+loopback
+lua
+markaby
+maruku
+matcher: catcher
+mediawiki
+memcache
 memoization
+memoize
 memoized
+merb
 metadata
+metaprogramming
 middleware: example
+minitest: example
+mins
 mixin: toxin
+modularity
+mongo
+mongoid
 monkeypatch: patch
 monkeypatching
+mtime
+multibyte
+multiline
 multithreaded
 multithreading
+mustermann
+mutator
 mutex: class
+namer
 namespace: example
+natively
+nginx
+nginx
+nils
+nodoc
+nokogiri: example
+nonces
+noop
+npm
+numericality
+offence
+openssl
+overridable
 override: ride
+overriden
+pandoc
+parallelize
 param: example
 parens
+parseable
 parser: example
+pathname
+pathname
+perf
+performant
+permalink: link
+pid
+pipelined
+pipelining
+plaintext
+pluggable
 plugin: penguin
+populator
+postfix
+postgres
 pre
+precompile
+precompiled
+preload
+preloaded
+preloader
+preloading
+preloads
 prepend: append
-proc
+preprocessing
+prerelease
+proc: example
+profiler
+programmatically
+proxied
+proxying
+pubsub
+pygments
+rack-contrib
+rackup
+ragel
+rails's
+railtie
+railties
+rbenv
+rbx
+rdoc
+readline
+readme
+readonly
+rebase
+reconfirmable
+redcarpet
+redistribution: distribution
 refactor
 refactoring
+refactorings
 regex
 regexp
-repo
+reloader: loader
+rememberable
+renderer: example
+repl
+repo: example
+representable
+representer
+repro
+reraise
+resque
+rhs
+roadmap
+rspec
+rspec-dev
+rspec-support
+rubinius
+rubocop
 rubygems
 runtime: example
+sanitization
+savepoint: point
+scalability
+schemas
+scss
+segv
+sendmail
+serializable
+serializer: example
+serializers
+servlet
+sexualized
+sharding
+sidekiq
+signalling
+simples
+singularized
+solaris
+src
+ssl
+stabby
+stateful
+stdcall
 stderr
 stdin
 stdlib
+stdlibs
 stdout
+str
+strftime
+stringified
+stringify
+stringify
+struct: structure
 stylesheet: sheet
 subclass: class
 subclassing
-subtype: example
+subcommand: commnad
+subdirectory: directory
+subfolder: folder
+sublicense: license
+subpath: path
+subprocess
+substring: string
+subtype: type
+sudo
 superclass: class
+symlink: link
+symlinked
+syntaxes
+sysrandom
+teardown
+tempfile: file
+templating
+textarea: area
+thoughtbot
+threadsafe
 timestamp: stamp
+timezone: zone
+tmp
+todo
+tokenize
+tokenized
 tokenizer: example
+toplevel
+trackable
+tradeoff
+transactional
+transcoding
+triaging
 truthy
+tt
+tty
+turbolinks
+typedef
+tz
+uglifier
+unary
+uncomment
 unescape
+unescaped
+unescapes
+unescaping
+unformatted
+unhandled
 unicode
+unindent
+uniq
+unlink: link
+unlinked
+unparenthesized
+unreferenced
+unregister
+unscoped
+untrusted
+uri
+usec
 username: example
+uuid
+valgrind
+validator: predator
+validators
+vendored
+vendoring
+verifier
+verifiers
+viewport
+webrick
+webserver
 whitespace: example
+wkhtmltopdf
+writeable
+wtf
+xmlns
+xpath
+yajl
+yaml
+yardoc
+yay
+zoneinfo

data/lib/forspell/runner.rb

@@ -10,8 +10,13 @@ module Forspell
     end
 
     def call
-      @files.each do |path|
+      increment = (@files.size / 100.0).ceil
+      total = @files.size <= 100 ? @files.size : 100
+      @reporter.progress_bar = ProgressBar.create(total: total, output: $stderr)
+
+      @files.each_with_index do |path, index|
         process_file path
+        @reporter.progress_bar.increment if (index + 1) % increment == 0
       end
 
       @reporter.report

data/lib/forspell/sanitizer.rb

@@ -5,18 +5,12 @@ require 'cgi'
 
 module Forspell
   module Sanitizer    
-    REMOVE_PUNCT = /[[:punct:]&&[^\-\'\_\.]]$/.freeze
+    REMOVE_PUNCT = %r{[!|?|,|(|)|"|;]+}.freeze
 
     def self.sanitize(input)
 
-      result = CGI.unescapeHTML(Sanitize.fragment(input,
-                                         elements: [], remove_contents: true))
-                .gsub(REMOVE_PUNCT, '').gsub(/[\!\.\?]{1}$/, '')
-      if result.start_with?("'") && result.end_with?("'")
-        result[1..-2]
-      else
-        result
+      CGI.unescapeHTML(Sanitize.fragment(input, elements: [], remove_contents: true))
+         .gsub(REMOVE_PUNCT, '').gsub(%r{[.:]+$}, '')
       end
-    end
   end
 end

data/lib/forspell/speller.rb

@@ -1,16 +1,17 @@
 # frozen_string_literal: true
 
 require 'ffi/hunspell'
+require 'backports/2.5.0/enumerable/all'
 
 module Forspell
   class Speller
     attr_reader :dictionary
 
-    SUGGESTIONS_SIZE = 3
     HUNSPELL_DIRS = [File.join(__dir__, 'dictionaries')]
     RUBY_DICT = File.join(__dir__, 'ruby.dict')
 
-    def initialize(main_dictionary, *custom_dictionaries)
+    def initialize(main_dictionary, *custom_dictionaries, suggestions_size: 0)
+      @suggestions_size = suggestions_size
       FFI::Hunspell.directories = HUNSPELL_DIRS << File.dirname(main_dictionary)
       @dictionary = FFI::Hunspell.dict(File.basename(main_dictionary))
 
@@ -28,11 +29,20 @@ module Forspell
     end
 
     def correct?(word)
-      dictionary.check?(word)
+      parts = word.split('-')
+      if parts.size == 1
+        alterations = [word]
+        alterations << word.capitalize unless word.capitalize == word
+        alterations << word.upcase unless word.upcase == word
+        
+        alterations.any?{ |w| dictionary.check?(w) }
+      else
+        parts.all? { |part| correct?(part) }
+      end
     end
 
     def suggest(word)
-      dictionary.suggest(word).first(SUGGESTIONS_SIZE)
+      @suggestions_size.positive? ? dictionary.suggest(word).first(@suggestions_size) : []
     end
   end
 end

data/lib/forspell/word_matcher.rb

@@ -3,12 +3,10 @@ require 'backports/2.4.0/regexp/match'
 module Forspell
   module WordMatcher
     WORD = %r{^
-      \'?                # could start with apostrophe
-      ([a-z]|[A-Z])?     # at least one letter,
-      ([[:lower:]])+     # then any number of letters,
-      ([\'\-])?          # optional dash/apostrophe,
-      ([[:lower:]])*     # another bunch of letters
-      \'?                # could end with apostrophe
+      ([a-z]|[A-Z])                 # at least one letter,
+      ([[:lower:]])*                # then any number of letters,
+      ([\'\-][[:lower:]])?          # optional dash/apostrophe, followed by letter
+      ([[:lower:]])*                # another bunch of letters
     $}x
 
     def self.word? text

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: forspell
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.7
 platform: ruby
 authors:
 - Kirill Kuprikov
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-05-26 00:00:00.000000000 Z
+date: 2019-05-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: slop
@@ -137,6 +137,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: highline
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby-progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement