LittleWeasel 3.0.3 → 5.0.0

data/Rakefile

@@ -1,13 +1,306 @@
-require "bundler/gem_tasks"
+# frozen_string_literal: true
+
+require 'active_support/core_ext/object/try.rb'
+require 'active_support/inflector'
+require 'benchmark/ips'
+require 'bundler/gem_tasks'
+require 'pry'
+
+require_relative 'lib/LittleWeasel'
+require_relative 'spec/support/file_helpers'
+require_relative 'spec/support/general_helpers'
+
+DictionaryResultsHelpers = Support::GeneralHelpers::DictionaryResultsHelpers
+
+def file_from(dictionary_key)
+  Support::FileHelpers.dictionary_path_for(file_name: dictionary_key.key)
+end
 
 begin
   require 'rspec/core/rake_task'
   RSpec::Core::RakeTask.new(:spec)
 rescue LoadError => e
-  task "spec" do
+  task 'spec' do
     puts "RSpec not loaded - make sure it's installed and you're using bundle exec"
     exit 1
   end
 end
 
-task :default => :spec
+#
+# Tasks related to the #word_results API
+
+namespace 'word_results' do
+  # Creates a dictionary from a file on disk
+  task :basic do
+    LittleWeasel.configure do |config|
+      # TODO: Configure as needed here.
+    end
+
+    # Create a Dictionary Manager.
+    dictionary_manager = LittleWeasel::DictionaryManager.new
+
+    # Create our unique key for the dictionary.
+    en_us_key = LittleWeasel::DictionaryKey.new(language: :en, region: :us)
+
+    file = Support::FileHelpers.dictionary_path_for file_name: en_us_key.key
+
+    # Create a dictionary of names from memory.
+    en_us_names_dictionary = dictionary_manager.create_dictionary_from_file(
+      dictionary_key: en_us_key,
+      file: file)
+
+    # Get some word results...
+
+    # Get results for a word we know exists.
+    word = 'apple'
+    word_results = en_us_names_dictionary.word_results word
+    DictionaryResultsHelpers.print_word_results word, word_results, "found (#{word} is in the dictionary)"
+
+    # Get results for a word we know DOES NOT exist.
+    word = 'dapple'
+    word_results = en_us_names_dictionary.word_results word
+    DictionaryResultsHelpers.print_word_results word, word_results, "not found (#{word} is not in the dictionary)"
+  rescue StandardError => e
+    task 'word_results:basic' do
+      puts "LittleWeasel task word_results:basic not loaded: #{e.message}"
+      exit 1
+    end
+  end
+
+  # Creates a dictionary of names from memory
+  task :from_memory do
+    LittleWeasel.configure do |config|
+      # TODO: Configure as needed here.
+    end
+
+    # Create a Dictionary Manager.
+    dictionary_manager = LittleWeasel::DictionaryManager.new
+
+    # Create our unique key for the dictionary.
+    en_us_names_key = LittleWeasel::DictionaryKey.new(language: :en, region: :us, tag: :names)
+
+    # Create a dictionary of names from memory.
+    en_us_names_dictionary = dictionary_manager.create_dictionary_from_memory(
+      dictionary_key: en_us_names_key, dictionary_words: %w(Abel Bartholomew Cain Deborah Elijah))
+
+    # Get some word results...
+
+    # Get results for a name we know exists.
+    word = 'Abel'
+    word_results = en_us_names_dictionary.word_results word
+    DictionaryResultsHelpers.print_word_results word, word_results, "found (#{word} is in the dictionary)"
+
+    # Get results for a name we know DOES NOT exist.
+    word = 'Henry'
+    word_results = en_us_names_dictionary.word_results word
+    DictionaryResultsHelpers.print_word_results word, word_results, "not found (#{word} is not in the dictionary)"
+  rescue StandardError => e
+    task 'word_results:from_memory' do
+      puts "LittleWeasel task word_results:from_memory not loaded: #{e.message}"
+      exit 1
+    end
+  end
+
+  # Shows application of word filters and word preprocessors.
+  task :advanced do
+    LittleWeasel.configure do |config|
+      # TODO: Configure as needed here.
+    end
+
+    # Create a Dictionary Manager.
+    dictionary_manager = LittleWeasel::DictionaryManager.new
+
+    # Create our unique key for the dictionary.
+    en_us_names_key = LittleWeasel::DictionaryKey.new(language: :en, region: :us, tag: :names)
+
+    # Create a Henry word filter.
+    class HenryFilter < LittleWeasel::Filters::WordFilter
+      class << self
+        def filter_match?(word)
+          word== 'Henry'
+        end
+      end
+    end
+    word_filters = [HenryFilter.new]
+
+    # Add a word preprocessor.
+    word_preprocessors = [LittleWeasel::Preprocessors::EnUs::CapitalizePreprocessor.new]
+
+    # Create a dictionary of names from memory.
+    en_us_names_dictionary = dictionary_manager.create_dictionary_from_memory(
+      dictionary_key: en_us_names_key,
+      dictionary_words: %w(Abel Bartholomew Cain Deborah Elijah),
+      word_filters: word_filters,
+      word_preprocessors: word_preprocessors)
+
+    puts '# Turning off our word filters and word preprocessors to start...'
+    puts
+
+    en_us_names_dictionary.filters_on = false
+    en_us_names_dictionary.preprocessors_on = false
+
+    # Get results for a name we know DOES NOT exist.
+    word = 'Henry'
+    word_results = en_us_names_dictionary.word_results word
+    DictionaryResultsHelpers.print_word_results word, word_results, "not found, #success? == false, word_valid? == false (#{word} is not in the dictionary)"
+
+    puts '# Turning word filters on...'
+    puts
+
+    en_us_names_dictionary.filters_on = true
+
+    # Get results for Henry again - it should be found due to the filter.
+    word = 'Henry'
+    word_results = en_us_names_dictionary.word_results word
+    DictionaryResultsHelpers.print_word_results word, word_results, '#success? == true due to the HenryFilter'
+
+    # Get results for a name we know DOES NOT exist.
+    word = 'henry'
+    word_results = en_us_names_dictionary.word_results word
+    DictionaryResultsHelpers.print_word_results word, word_results, "not found, #success? == false (#{word} is not in the dictionary and henry is lower case, no filter match)"
+
+    puts '# Turning preprocessors on so that henry is converted to Henry '
+    puts "# and consequently, the filter will match..."
+    puts
+
+    en_us_names_dictionary.preprocessors_on = true
+
+    word = 'henry'
+    word_results = en_us_names_dictionary.word_results word
+    DictionaryResultsHelpers.print_word_results word, word_results, "#success? == true, #filter_match? == true (#{word} is not in the dictionary but the word preprocessor and word filter work together to get a filter match and consider the name valid)"
+  rescue StandardError => e
+    task 'word_results:advanced' do
+      puts "LittleWeasel task word_results:advanced not loaded: #{e.message}"
+      exit 1
+    end
+  end
+
+  task :word_filters do
+    LittleWeasel.configure do |config|
+      # TODO: Configure as needed here.
+    end
+    dictionary_manager = LittleWeasel::DictionaryManager.new
+    dictionary_key = LittleWeasel::DictionaryKey.new(language: :en, region: :us)
+    file = Support::FileHelpers.dictionary_path_for file_name: dictionary_key.key
+    word_filters = [
+      LittleWeasel::Filters::EnUs::NumericFilter.new,
+      LittleWeasel::Filters::EnUs::CurrencyFilter.new,
+      LittleWeasel::Filters::EnUs::SingleCharacterWordFilter.new
+    ]
+    word_preprocessors = nil
+    dictionary_words = Support::FileHelpers.dictionary_words_for dictionary_file_path: file
+    dictionary = dictionary_manager.create_dictionary_from_memory(dictionary_key: dictionary_key, dictionary_words: dictionary_words, word_filters: word_filters, word_preprocessors: word_preprocessors)
+    dictionary_words << 'A'.dup
+    dictionary_words << 'I'.dup
+    dictionary_words << '1000'.dup
+    dictionary_words << '1,000'.dup
+    dictionary_words << '10,000.00'.dup
+    dictionary_words << '+100.00'.dup
+    dictionary_words << '-200,000.00'.dup
+    dictionary_words << '$100,000'.dup
+    dictionary_words << '+$100,000,000.10'.dup
+    dictionary_words << '-$999,000,000.10'.dup
+    dictionary_words.each do |word|
+      word.strip!
+      word_results = dictionary.word_results word
+      DictionaryResultsHelpers.print_word_results word, word_results
+    end
+  rescue StandardError => e
+    task 'word_results:word_filters' do
+      puts "LittleWeasel task word_results:word_filters not loaded: #{e.message}"
+      exit 1
+    end
+  end
+end
+
+#
+# Tasks related to the #block_results API
+
+namespace 'block_results' do
+  task :basic do
+    LittleWeasel.configure do |config|
+      # TODO: Configure as needed here.
+    end
+
+    # Create a Dictionary Manager.
+    dictionary_manager = LittleWeasel::DictionaryManager.new
+
+    # Create our unique key for the dictionary.
+    en_us_key = LittleWeasel::DictionaryKey.new(language: :en, region: :us, tag: :big)
+
+    # Create a dictionary from a file on disk. The below assumes the
+    # dictionary file name matches the dictionary key (e.g. en-US-big).
+    en_us_dictionary = dictionary_manager.create_dictionary_from_file(
+      dictionary_key: en_us_key, file: file_from(en_us_key))
+
+    word_block = "This is a word-block of 8 words and 2 numbers."
+
+    # Add a word filter so that numbers are considered valid.
+    en_us_dictionary.add_filters word_filters: [
+      LittleWeasel::Filters::EnUs::NumericFilter.new
+    ]
+
+    block_results = en_us_dictionary.block_results word_block
+
+    # Returns a LittleWeasel::BlockResults object.
+    DictionaryResultsHelpers.print_block_results word_block, block_results
+  rescue StandardError => e
+    task 'block_results:basic' do
+      puts "LittleWeasel task block_results:basic not loaded: #{e.message}"
+      exit 1
+    end
+  end
+end
+
+namespace :bm do
+  task :hash do
+    STRING_LOCALE = { 'en-US' => 'en-us' }
+    SYMBOL_LOCALE = { 'en-US' => :enUS }
+
+    puts 'String variable vs. normal String.'
+    Benchmark.ips do |x|
+      string_variable = 'string_variable'
+      x.report('string variable') { STRING_LOCALE[string_variable] }
+      x.report('normal') { STRING_LOCALE['en-US'] }
+    end
+
+    puts 'String#freeze vs. normal String.'
+    Benchmark.ips do |x|
+      x.report('freeze') { STRING_LOCALE['en-US'.freeze] }
+      x.report('normal') { STRING_LOCALE['en-US'] }
+    end
+
+    puts 'String vs Symbol'
+    Benchmark.ips do |x|
+      x.report('string') { STRING_LOCALE['en-US'] }
+      x.report('symbol') { SYMBOL_LOCALE[:enUS] }
+    end
+
+    puts 'String#freeze vs. Symbol'
+    Benchmark.ips do |x|
+      x.report('string') { STRING_LOCALE['en-US'.freeze] }
+      x.report('symbol') { SYMBOL_LOCALE[:enUS] }
+    end
+  rescue StandardError => e
+    task 'hash' do
+      puts "LittleWeasel task bm:hash not loaded: #{e.message}"
+      exit 1
+    end
+  end
+
+  task :dictionary_key do
+    puts 'DictionaryKey test'
+    Benchmark.ips do |x|
+      x.report('DictionaryKey') do
+        DictionaryKey.key(language: :en, region: :us, tag: :tag)
+      end
+    end
+  rescue StandardError => e
+    task 'locale' do
+      puts "LittleWeasel task bm:dictionary_key not loaded: #{e.message}"
+      exit 1
+    end
+  end
+end
+
+task default: :spec

data/lib/LittleWeasel/block_results.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require 'active_support/core_ext/module/delegation'
+require_relative 'word_results'
+
+module LittleWeasel
+  # This class represents the results of gathering information about a word
+  # block (group of words).
+  class BlockResults
+    # :reek:Attribute - Ignored, it doesn't make sense to create a formal setter method.
+    attr_reader :original_word_block, :word_results
+
+    def initialize(original_word_block:)
+      self.original_word_block = original_word_block
+      self.word_results = []
+    end
+
+    def <<(word_result)
+      unless word_result.is_a? WordResults
+        raise ArgumentError, "Argument word_result is not a WordResults object: #{word_result.class}"
+      end
+
+      word_results << word_result
+    end
+
+    # Calls #success? on all WordResults objects. Returns true if all
+    # WordResults return true; false is returned otherwise.
+    def success?
+      return false unless word_results.present?
+
+      word_results.all?(&:success?)
+    end
+
+    # Returns true if all WordResults object words are valid (#word_valid?);
+    # false otherwise.
+    def words_valid?
+      return false unless word_results.present?
+
+      word_results.all?(&:word_valid?)
+    end
+
+    # Returns true if all WordResults object words have filter matches (#filters_match?);
+    # false otherwise.
+    def filters_match?
+      return false unless word_results.present?
+
+      word_results.all?(&:filter_match?)
+    end
+
+    # Returns true if all WordResults object words have been preprocessed (#preprocessed_words?);
+    # false otherwise.
+    def preprocessed_words?
+      return false unless word_results.present?
+
+      word_results.all?(&:preprocessed_word?)
+    end
+
+    # Returns an Array of the results of calling
+    # #preprocessed_word_or_original_word on all WordResults objects.
+
+    # Calls #preprocessed_word_or_original_word on all WordResults objects.
+    # An Array of the results is returned.
+    def preprocessed_words_or_original_words
+      return [] unless word_results.present?
+
+      word_results.map(&:preprocessed_word_or_original_word)
+    end
+
+    # Returns true if all WordResults object words have been cached (#words_cached?);
+    # false otherwise.
+    def words_cached?
+      return false unless word_results.present?
+
+      word_results.all?(&:word_cached?)
+    end
+
+    private
+
+    attr_writer :original_word_block, :word_results
+  end
+end

data/lib/LittleWeasel/configure.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+# This is the configuration for LittleWeasel.
+module LittleWeasel
+  class << self
+    attr_reader :configuration
+
+    # Returns the application configuration object.
+    #
+    # @return [Configuration] the application Configuration object.
+    def configure
+      self.configuration ||= Configuration.new
+      yield(configuration)
+    end
+
+    private
+
+    attr_writer :configuration
+  end
+
+  # This class encapsulates the configuration properties for this gem and
+  # provides methods and attributes that allow for management of the same.
+  #
+  # attr_reader :max_dictionary_file_megabytes, :max_invalid_words_bytesize, :metadata_observers
+  class Configuration
+    attr_reader :max_dictionary_file_megabytes,
+      :max_invalid_words_bytesize, :metadata_observers, :word_block_regex
+
+    # The constructor; calls {#reset}.
+    def initialize
+      reset
+    end
+
+    # Resets the configuration settings to their default values.
+    #
+    # @return [void]
+    def reset
+      @max_dictionary_file_megabytes = 5
+      @max_invalid_words_bytesize = 25_000
+      @metadata_observers = [
+        LittleWeasel::Metadata::InvalidWordsMetadata
+      ]
+      # TODO: Is this the correct regex to use, or is there something better?
+      # @word_block_regex = /\s+(?=(?:[^"]*"[^"]*")*[^"]*$)/
+      # @word_block_regex = /(?:(?:[\-A-Za-z0-9]|\d(?!\d|\b))+(?:'[\-A-Za-z0-9]+)?)/
+      # @word_block_regex = /(?:(?:[\-a-z0-9]|\d(?!\d|\b))+(?:'[\-a-z0-9]+)?)/i
+      @word_block_regex = /[[[:word:]]'-]+/
+    end
+
+    # Returns the maximum consumable dictionary size in bytes. Dictionaries
+    # larger than {#max_dictionary_file_bytes} will raise an error.
+    #
+    # The default is 5 megabytes.
+    #
+    # @return [Integer] the maximum number of bytes for a dictionary.
+    def max_dictionary_file_bytes
+      @max_dictionary_file_megabytes * 1_000_000
+    end
+
+    # If {#max_invalid_words_bytesize} is > 0, true will be returned; false
+    # otherwise.
+    #
+    # @return [true, false] based on {#max_invalid_words_bytesize}.
+    def max_invalid_words_bytesize?
+      max_invalid_words_bytesize.positive?
+    end
+
+    # rubocop: disable Style/TrivialAccessors
+    def max_dictionary_file_megabytes=(value)
+      @max_dictionary_file_megabytes = value
+    end
+
+    # Sets the maximum cache size (in bytes) for invalid words. If
+    # less than or equal to 0, invalid words will NOT be cached.
+    #
+    # If greater than 0, invalid words will be cached up to and including
+    # {#max_invalid_words_bytesize} bytes.
+    #
+    # @see #max_invalid_words_bytesize?
+    def max_invalid_words_bytesize=(value)
+      value = 0 if value.negative?
+      @max_invalid_words_bytesize = value
+    end
+
+    def metadata_observers=(value)
+      raise ArgumentError, "Argument value is not an Array: #{value.class}" unless value.is_a? Array
+
+      # TODO: Limit the amount of observer classes, exploits?
+
+      @metadata_observers = value
+    end
+
+    def word_block_regex=(value)
+      @word_block_regex = value
+    end
+    # rubocop: enable Style/TrivialAccessors
+  end
+end

data/lib/LittleWeasel/dictionary.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+require_relative 'block_results'
+require_relative 'filters/word_filter_managable'
+require_relative 'metadata/dictionary_metadata'
+require_relative 'modules/configurable'
+require_relative 'modules/dictionary_cache_servicable'
+require_relative 'modules/dictionary_keyable'
+require_relative 'modules/dictionary_metadata_servicable'
+require_relative 'preprocessors/word_preprocessor_managable'
+require_relative 'word_results'
+
+module LittleWeasel
+  class Dictionary
+    include Filters::WordFilterManagable
+    include Modules::Configurable
+    include Modules::DictionaryCacheServicable
+    include Modules::DictionaryKeyable
+    include Modules::DictionaryMetadataServicable
+    include Preprocessors::WordPreprocessorManagable
+
+    attr_reader :dictionary_metadata_object, :dictionary_words
+
+    def initialize(dictionary_key:, dictionary_words:, dictionary_cache:,
+      dictionary_metadata:, word_filters: nil, word_preprocessors: nil)
+      validate_dictionary_key dictionary_key: dictionary_key
+      self.dictionary_key = dictionary_key
+
+      validate_dictionary_cache dictionary_cache: dictionary_cache
+      self.dictionary_cache = dictionary_cache
+
+      validate_dictionary_metadata dictionary_metadata: dictionary_metadata
+      self.dictionary_metadata = dictionary_metadata
+
+      unless dictionary_words.is_a?(Array)
+        raise ArgumentError,
+          "Argument dictionary_words is not an Array: #{dictionary_words.class}"
+      end
+
+      # Set up the dictionary metadata object and observers
+      self.dictionary_words = self.class.to_hash(dictionary_words: dictionary_words)
+      self.dictionary_metadata_object = create_dictionary_metadata
+      dictionary_metadata_object.add_observers
+
+      add_filters word_filters: word_filters || []
+      add_preprocessors word_preprocessors: word_preprocessors || []
+    end
+
+    class << self
+      def to_hash(dictionary_words:)
+        dictionary_words.each_with_object(Hash.new(false)) { |word, hash| hash[word] = true; }
+      end
+    end
+
+    def word_results(word)
+      # TODO: Make max word size configurable.
+      raise ArgumentError, "Argument word is not a String: #{word.class}" unless word.is_a?(String)
+
+      preprocessed_words = preprocess(word: word)
+      preprocessed_word = preprocessed_words.preprocessed_word
+      filters_matched = filters_matched(preprocessed_word || word)
+      word_results = WordResults.new(original_word: word,
+        filters_matched: filters_matched,
+        preprocessed_words: preprocessed_words,
+        word_cached: dictionary_words.include?(preprocessed_word || word),
+        word_valid: dictionary_words[preprocessed_word || word] || false)
+
+      dictionary_metadata_object.notify(action: :word_search,
+        params: { word_results: word_results })
+
+      word_results
+    end
+
+    def block_results(word_block)
+      # TODO: Make max word_block size configurable.
+      raise ArgumentError, "Argument word_block is not a String: #{word_block.class}" unless word_block.is_a?(String)
+      raise ArgumentError, "Argument word_block is empty: #{word_block.class}" unless word_block.present?
+
+      BlockResults.new(original_word_block: word_block).tap do |block_results|
+        word_block.scan(config.word_block_regex)&.map do |word|
+          block_results << word_results(word)
+        end
+      end
+    end
+
+    # This method returns true if this dictionary object is detached from the
+    # dictionary cache; this can happen if the dictionary object is unloaded
+    # from the dictionary cache(DictionaryManager#unload_dictionary) or the
+    # dictionary is killed (DictionaryManager#kill_dictionary).
+    def detached?
+      !dictionary_cache_service.dictionary_object?
+    end
+
+    # This method returns a count of VALID words in the dictionary.
+    def count
+      dictionary_words.each_pair.count { |_word, valid| valid }
+    end
+
+    # This method returns a count of all VALID and INVALID words in
+    # the dictionary.
+    def count_all_words
+      dictionary_words.count
+    end
+
+    # This method returns a count of all INVALID words in the dictionary.
+    def count_invalid_words
+      dictionary_words.each_pair.count { |_word, valid| !valid }
+    end
+
+    private
+
+    attr_writer :dictionary_metadata_object, :dictionary_words
+
+    def create_dictionary_metadata
+      # We unconditionally attach metadata to this dictionary. DictionaryMetadata
+      # only attaches the metadata services that are turned "on".
+      Metadata::DictionaryMetadata.new(
+        dictionary_words: dictionary_words,
+        dictionary_key: dictionary_key,
+        dictionary_cache: dictionary_cache,
+        dictionary_metadata: dictionary_metadata
+      )
+    end
+  end
+end

data/lib/LittleWeasel/dictionary_key.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require_relative 'modules/language_validatable'
+require_relative 'modules/locale'
+require_relative 'modules/region_validatable'
+require_relative 'modules/taggable'
+
+module LittleWeasel
+  # This class describes a unique key associated with a particular dictionary
+  # file. Dictionary keys are used to identify a dictionary on which an action
+  # should be performed.
+  class DictionaryKey
+    include Modules::LanguageValidatable
+    include Modules::Locale
+    include Modules::RegionValidatable
+    include Modules::Taggable
+
+    attr_reader :language, :region
+
+    def initialize(language:, region: nil, tag: nil)
+      validate_language language: language
+      self.language = normalize_language language
+
+      validate_region region: region
+      self.region = normalize_region region
+
+      validate_tag tag: tag
+      self.tag = tag
+    end
+
+    def key
+      return locale unless tagged?
+
+      "#{locale}-#{tag}"
+    end
+    alias to_s key
+
+    class << self
+      def key(language:, region: nil, tag: nil)
+        new(language: language, region: region, tag: tag).key
+      end
+    end
+
+    private
+
+    attr_writer :language, :region
+  end
+end