clausewitz-spelling 0.1.19 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 55b8f238c7ccf5a99e18aea147d865baa3eba23e
-  data.tar.gz: 737b15a9ea8c6dfb3b1279d1170986aa9b080e8f
+SHA256:
+  metadata.gz: c3f825db29cf16aa71180113fdba663b417a44cab29a2cd9d4c2a447c3061ae3
+  data.tar.gz: ae39cd480cffe2f0a3ff6cb673c00c53d0d86def3136b274806dc9cacadff860
 SHA512:
-  metadata.gz: 523a1c3c5cbed97de4454eca5422a181846544338e7fc108457f924f2a9a3ba9f3b1b1329679e5358f958b12b10d94972bc6a05767fc5feacf7b9bb804444ce5
-  data.tar.gz: 3206f38862471223cc8fd4096cb398b1b42864b15ef8a60bb82de1e507f4bf497bfb86e7e46fbbe57d8eb643141f18d4e42c2f22940314e36c3e6b82feaa4cf6
+  metadata.gz: e1880d26258c57c61d8ba8a1b9b3177e998657cef265411a09c9f5fb95616e370386e3286112e468af581ea8cda235cd4483e95e8c525303598d9377dff363e0
+  data.tar.gz: afcd28a0b5f6250b5d4d2f6ec90b443feda4017be1543fb2466f485b261c1b256cd2b6875c26fac95e23d5c571e4a000b95d2738501b78be78c88ee2649ea301

data/Gemfile.lock

@@ -1,10 +1,10 @@
 PATH
   remote: .
   specs:
-    clausewitz-spelling (0.1.19)
+    clausewitz-spelling (0.2.0)
       colorize
       damerau-levenshtein
-      ffi-aspell
+      ffi-hunspell-wtchappell
       optimist
       pragmatic_tokenizer
 
@@ -16,8 +16,8 @@ GEM
     damerau-levenshtein (1.3.1)
     diff-lcs (1.3)
     ffi (1.10.0)
-    ffi-aspell (1.1.0)
-      ffi
+    ffi-hunspell-wtchappell (0.4.0)
+      ffi (~> 1.0)
     method_source (0.9.2)
     optimist (3.0.0)
     pragmatic_tokenizer (3.0.7)
@@ -45,11 +45,11 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
-  bundler (~> 1.16)
+  bundler (~> 1.17.2)
   clausewitz-spelling!
   pry
   rake (~> 10.0)
   rspec (~> 3.0)
 
 BUNDLED WITH
-   1.16.1
+   1.17.2

data/clausewitz-spelling.gemspec

@@ -20,12 +20,12 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.16"
+  spec.add_development_dependency "bundler", "~> 1.17.2"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "pry"
 
-  spec.add_dependency "ffi-aspell"
+  spec.add_dependency "ffi-hunspell-wtchappell"
   spec.add_dependency "optimist"
   spec.add_dependency "colorize"
   spec.add_dependency "damerau-levenshtein"

data/en_GB_owb.dic

@@ -0,0 +1,81 @@
+35
+Adytum/1
+Almirante/1
+APC/2 1
+Allgood/1
+Aradesh/1
+Bearport/1
+Boneyard/1
+Chico/1
+Ciudad/1
+Colibrí/1
+Darkwater/1
+Dayglow/1
+Elijah/1
+Elijah's
+FEV/1
+GAI/1
+Gizmo/1
+Harbor/1
+Hardin/2 1
+Hardin's
+Harlon/1
+Héctor/1
+Henderton/1
+Hubology
+Hubologist/2 1
+Junktown/1
+Klamath/1
+Los
+Maxson/1
+Morbid/1
+Mossman/1
+NCR/1
+Paz/1
+Pesca/1
+Peterson/1
+Petro/1
+Rafael/1
+Rattletail/1
+Reynosa/1
+RobCo/1
+Santángel/1
+Seabear/1
+Shi/1
+Silus/1
+Tamaulipas/1
+Tampico/1
+Tandi/1
+Tech
+Tlaloc/1
+TODO/1
+West-Tek/1
+Valles/1
+Vandenberg/1
+Vault-Tec/1
+bandito/2 1
+brahmin/5 1
+caudillo/2 1
+commonfolk/5 1
+deathclaw/2 1
+doodad/2 1
+eyebot/2
+firefight/2 1
+firepower/5 1
+hardline/2 1
+lakebed/2 1
+malcontents
+military's
+playbook/2 1
+protectron/2 1
+radscorpion/2 1
+robobrain/2 1
+securitron/2 1
+shopfront/2 1
+superweapon/2 1
+tribals/5	1
+underway
+unpowered
+vertibird/2 1
+weaponsmith/2 1
+wildcard/2 1

data/exe/clausewitz-spellcheck

@@ -9,8 +9,11 @@ class Main
 
   def parse_args(args)
     opts = Optimist::options(args) do
-      opt :dictionary_root,
-        "Directory containing per-language custom word lists",
+      opt :custom_dicts,
+        "List of custom dictionaries to load",
+        type: :strings
+      opt :custom_dict_root,
+        "Directory containing per-language-dialect custom dicts",
         type: :string
       opt :suggestion_count,
         "How many suggestions to display",
@@ -21,7 +24,23 @@ class Main
           type: :string
       end
     end
-    [opts, args]
+
+    dialect_map = {}
+    dialect_opts = opts.keys.select { |k| k =~ /.+_dialect/ }
+    dialect_opts.each do |dialect_opt_key|
+      next unless opts[dialect_opt_key]
+      next if dialect_opt_key.to_s.end_with?('given')
+      language_name = dialect_opt_key[/(.+)_dialect/, 1]
+      dialect_map[language_name] = opts[dialect_opt_key]
+    end
+
+    checker_opts = {}
+    checker_opts[:custom_dict_root] = opts[:custom_dict_root]
+    checker_opts[:custom_dicts]     = opts[:custom_dicts]
+    checker_opts[:dialect_map]      = dialect_map
+    checker_opts[:suggestion_count] = opts[:suggestion_count]
+
+    [checker_opts, args]
   end
 
   def run

data/lib/clausewitz/localisation.rb

@@ -17,7 +17,7 @@ module Clausewitz
       end
 
       def select_dialect(dialect)
-        if @dialects.include?(dialect)
+        if @dialects.include?(dialect.downcase)
           @selected_dialect = dialect
         else
           fail("Unknown dialect override '#{dialect}'!")
@@ -48,11 +48,11 @@ module Clausewitz
       ),
       'l_spanish' => LangConfig.new(
         'spanish',
-        'es', []
+        'es', %w[mx es], 'es'
       ),
       'l_russian' => LangConfig.new(
         'russian',
-        'ru', []
+        'ru', %w[ru], 'ru'
       )
     }
 

data/lib/clausewitz/spelling/checker.rb

@@ -1,4 +1,4 @@
-require 'ffi/aspell'
+require 'ffi/hunspell'
 require 'open3'
 require 'pathname'
 require 'set'
@@ -11,27 +11,39 @@ require 'clausewitz/spelling/results'
 
 module Clausewitz; module Spelling
   class Checker
-    attr_accessor :dict_words
+    DEFAULT_SUGGESTION_COUNT = 3
+
     def initialize(opts = {})
-      @dictionary_root  = opts[:dictionary_root]
-      @suggestion_count = opts[:suggestion_count] || 3
+      @custom_dict_root = opts[:custom_dict_root]
+      @custom_dict_root = Pathname.new(@custom_dict_root) if @custom_dict_root
+      @custom_dicts     = opts[:custom_dicts]     || []
+      @dialect_map      = opts[:dialect_map]      || {}
+      @suggestion_count = opts[:suggestion_count] || DEFAULT_SUGGESTION_COUNT
 
-      if @dictionary_root
-        @dictionary_root = Pathname.new(@dictionary_root)
-      end
+      load_dictionaries!
+    end
 
-      dialect_opts = opts.keys.select { |k| k =~ /.+_dialect/ }
-      dialect_opts.each do |dialect_opt_key|
-        next unless opts[dialect_opt_key]
-        next if dialect_opt_key.to_s.end_with?('given')
-        language_name = dialect_opt_key[/(.+)_dialect/, 1]
-        config = language_config(language_name)
-        dialect = opts[dialect_opt_key]
-        config.select_dialect(dialect.downcase)
-      end
+    def load_dictionaries!
+      @loaded_dicts = {}
+      Localisation::LANG_MAP.each do |_, config|
+        if @dialect_map.key?(config.name)
+          config.select_dialect(@dialect_map[config.name])
+        end
+
+        dict = FFI::Hunspell.dict(config.full_name)
+
+        @custom_dicts.each do |custom_dict|
+          path = @custom_dict_root.join("#{config.full_name}_#{custom_dict}")
+          path = Pathname.new("#{path}.dic")
+          if path.exist?
+            dict.add_dic(path.to_s)
+          else
+            $stderr.puts("Could not load dictionary '#{path}', skipping...")
+          end
+        end
 
-      @loaded_spellcheckers = {}
-      @loaded_wordlists     = {}
+        @loaded_dicts[config.name] = dict
+      end
     end
 
     def check_file(filepath)
@@ -43,7 +55,7 @@ module Clausewitz; module Spelling
         return InvalidFilepathResult.new(filepath, e)
       end
 
-      $stderr.puts "Skipping #{filepath}..." if filepath.directory?
+      $stderr.puts "Skipping directory '#{filepath}'..." if filepath.directory?
 
       begin
         contents = Clausewitz::Localisation.parse_file(filepath)
@@ -53,16 +65,14 @@ module Clausewitz; module Spelling
 
       checks = contents.map do |lang_name, entries|
         lc = language_config(lang_name)
-        check_entries(lc, entries)
+        check_entries(entries, lc)
       end
       FileResults.new(filepath, checks)
     end
 
     private
 
-    def check_entries(lc, entries)
-      wordlist = load_wordlist(lc)
-      aspell_checker = load_aspell_checker(lc)
+    def check_entries(entries, lc)
       spellcheck_ignore = entries&.delete('spellcheck_ignore')
       ignored_keys = spellcheck_ignore ? spellcheck_ignore.split(',') : []
       ignored_keys << 'spellcheck_ignore'
@@ -74,13 +84,13 @@ module Clausewitz; module Spelling
         if ignored_keys.include?(key)
           IgnoredEntryResult.new(key)
         else
-          check_entry(lc, aspell_checker, wordlist, key, entry)
+          check_entry(key, entry, lc)
         end
       end
       LangResults.new(lc.clausewitz_name, checks)
     end
 
-    def check_entry(lc, checker, wordlist, key, entry)
+    def check_entry(key, entry, lc)
       # We don't want to pay attention to scripted localisation, so we'll strip
       # it out before we start.
       # TODO: Look into supporting escaped square brackets as part of the
@@ -123,13 +133,19 @@ module Clausewitz; module Spelling
       }
       words = PragmaticTokenizer::Tokenizer.new(opts).tokenize(entry)
       words = words.map { |word| word.split('—') }.flatten(1)
+      words.map! do |word|
+        if word =~ /[[:alpha:]]\.$/ && word.chars.count('.') == 1
+          word.sub(/\.$/, '')
+        else
+          word
+        end
+      end
 
-
-      checks = words.map { |word| check_word(checker, wordlist, word) }.compact
+      checks = words.map { |word| check_word(word, lc) }.compact
       EntryResults.new(key, checks)
     end
 
-    def check_word(checker, wordlist, word)
+    def check_word(word, lc)
       return if is_number?(word)
       return if is_plural_number?(word)
       return if is_ordinal?(word)
@@ -137,10 +153,11 @@ module Clausewitz; module Spelling
       return if is_icon?(word)
       return if is_initial?(word)
       return if is_psalm?(word)
-      return if wordlist.include?(word)
 
-      if !checker.correct?(word)
-        MisspelledWordResult.new(word, suggest_words(checker, wordlist, word))
+      lang_dict = @loaded_dicts[lc.name]
+      if !lang_dict.check?(word)
+        suggestions = lang_dict.suggest(word).take(@suggestion_count)
+        MisspelledWordResult.new(word, suggestions)
       end
     end
 
@@ -156,26 +173,6 @@ module Clausewitz; module Spelling
       word =~ /^[A-Z]\.$/
     end
 
-    def suggest_words(checker, wordlist, word)
-      return [] if word.size < 3
-
-      suggestions = Set.new
-
-      aspell_suggestions = checker.suggestions(word)
-
-      custom_suggestions = wordlist.select do |dict_word|
-        min = [word.size, dict_word.size].min
-        DamerauLevenshtein.distance(word, dict_word) < min
-      end
-
-      aspell_suggestions.each { |sug| suggestions.add(sug) }
-      custom_suggestions.each { |sug| suggestions.add(sug) }
-
-      suggestions.to_a.sort_by do |sug|
-        DamerauLevenshtein.distance(sug, word)
-      end.first(@suggestion_count)
-    end
-
     def is_icon?(word)
       word =~ /^£\w+/
     end
@@ -193,71 +190,19 @@ module Clausewitz; module Spelling
         word =~ /%(-|\+)?[0-9]+(\.[0-9]+)?/
     end
 
-    # Loads our custom wordlist into a temporary Aspell dictionary.
-    # This way Aspell won't yell at us for custom words and will also
-    # potentially select from this list as suggestions for misspelled words.
-    def load_custom_dictionary(lc)
-      dir = Dir.mktmpdir("custom-wordlist-#{lc.full_name}-")
-      output = File.join(dir, "#{lc.full_name}-custom.wlst")
-      cmd = %W[
-        aspell --lang=#{lc.base} --encoding=UTF-8 create master #{output}
-      ]
-      value = nil
-      Open3.popen3(*cmd) do |stdin, stdout, stderr, wait_thr|
-        dict_path = File.join(@dictionary_root, lc.full_name, 'dict.txt')
-				contents = File.read(dict_path)
-				words = contents.lines.map(&:chomp)
-        words.each do |word|
-          stdin.puts(word)
-        end
-        stdin.close
-        value = wait_thr.value
-      end
-			unless value.success?
-      	fail("Could not generate custom word list for #{lc.full_name}!")
-			end
-      output
-    end
-
-    def load_aspell_checker(lc)
-      if @loaded_spellcheckers[lc.full_name]
-        return @loaded_spellcheckers[lc.full_name]
-      end
-      aspell_checker = FFI::Aspell::Speller.new(
-        lc.full_name, encoding: 'UTF-8'
-      )
-			aspell_checker.set('ignore-accents', true)
-      if @dictionary_root && @dictionary_root.join(lc.full_name).exist?
-        custom_words = load_custom_dictionary(lc)
-				aspell_checker.set('extra-dicts', custom_words)
-      end
-			@loaded_spellcheckers[lc.full_name] = aspell_checker
-    end
-
-    def load_wordlist(lc)
-      return @loaded_wordlists[lc.full_name] if @loaded_wordlists[lc.full_name]
-      contents = ''
-      dict_path = @dictionary_root.join(lc.full_name, 'dict.txt')
-      if @dictionary_root && dict_path.exist?
-				contents = File.read(@dictionary_root.join(lc.full_name, 'dict.txt'))
-      end
-      words = contents.lines.to_a.map(&:chomp)
-			@loaded_wordlists[lc.full_name] = Set.new(words)
-    end
-
     def language_config(language_name)
       language_name = "l_#{language_name}" if language_name !~ /^l_/
-      aspell_lang_config = Localisation::LANG_MAP.find do |config_key, _|
+      lang_config = Localisation::LANG_MAP.find do |config_key, _|
         language_name == config_key
       end
-      fail("Unknown language '#{language_name}'!") unless aspell_lang_config
-      aspell_lang_config.last
+      fail("Unknown language '#{language_name}'!") unless lang_config
+      lang_config.last
     end
 
     # Make sure a file to be checked is actually present and readable.
     def validate_filepath!(filepath)
-      fail("No such file #{filepath}!") unless filepath.exist?
-      fail("Cannot read #{filepath}!")  unless filepath.readable?
+      fail("No such file '#{filepath}'!") unless filepath.exist?
+      fail("Cannot read '#{filepath}'!")  unless filepath.readable?
     end
   end
 end; end

data/lib/clausewitz/spelling/version.rb

@@ -1,5 +1,5 @@
 module Clausewitz
   module Spelling
-    VERSION = "0.1.19"
+    VERSION = "0.2.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: clausewitz-spelling
 version: !ruby/object:Gem::Version
-  version: 0.1.19
+  version: 0.2.0
 platform: ruby
 authors:
 - Will Chappell
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-01-15 00:00:00.000000000 Z
+date: 2019-06-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: 1.17.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: 1.17.2
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -67,7 +67,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: ffi-aspell
+  name: ffi-hunspell-wtchappell
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -157,6 +157,7 @@ files:
 - bin/console
 - bin/setup
 - clausewitz-spelling.gemspec
+- en_GB_owb.dic
 - exe/clausewitz-spellcheck
 - lib/clausewitz/localisation.rb
 - lib/clausewitz/spelling.rb
@@ -182,8 +183,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.2.1
+rubygems_version: 3.0.1
 signing_key: 
 specification_version: 4
 summary: Spellchecker tool for Clausewitz engine files