sastrawi-ruby 0.2.0 → 0.3.0

data/lib/sastrawi/dictionary/array_dictionary.rb

@@ -1,14 +1,23 @@
+# frozen_string_literal: true
+
+require 'set'
+
 module Sastrawi
   module Dictionary
     class ArrayDictionary
-      attr_reader :words
-
       def initialize(words = [])
-        @words = []
+        @words = Set.new
 
         add_words(words)
       end
 
+      ##
+      # Return the words as an Array (for backward compatibility)
+
+      def words
+        @words.to_a
+      end
+
       ##
       # Check whether a word is contained in the dictionary
 
@@ -20,7 +29,7 @@ module Sastrawi
       # Count how many words in the dictionary
 
       def count
-        @words.length
+        @words.size
       end
 
       ##
@@ -36,9 +45,13 @@ module Sastrawi
       # Add a word to the dictionary
 
       def add(word)
-        return if word.nil? || word.strip == ''
+        unless word.is_a?(String)
+          raise ArgumentError, "dictionary entries must be strings, got #{word.class}"
+        end
+
+        return if word.strip == ''
 
-        @words.push(word)
+        @words.add(word)
       end
 
       ##

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule10.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule11.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule12.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule13a.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule13b.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule14.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule15a.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule15b.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule16.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule17a.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule17b.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule17c.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule17d.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule18a.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule18b.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule19.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule1a.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule1b.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule2.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule20.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule21a.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule21b.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule23.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule24.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule25.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule26a.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule26b.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule27.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule28a.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule28b.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule29.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule3.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule30a.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule30b.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule30c.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule31a.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule31b.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule32.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule34.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule35.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule36.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule37a.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule37b.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule38a.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule38b.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule39a.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule39b.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule4.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule40a.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule40b.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule41.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule42.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule5.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule6a.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule6b.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule7.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule8.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule9.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Morphology
     module Disambiguator

data/lib/sastrawi/morphology/invalid_affix_pair_specification.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 ##
 # Asian J. (2007) "Effective Techniques for Indonesian Text Retrieval". page 26
 # http://researchbank.rmit.edu.au/eserv/rmit:6312/Asian.pdf

data/lib/sastrawi/stemmer/cache/array_cache.rb

@@ -1,23 +1,58 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Stemmer
     module Cache
       class ArrayCache
-        attr_reader :data
+        DEFAULT_MAX_SIZE = 10_000
+
+        attr_reader :max_size
 
-        def initialize
+        def initialize(max_size: DEFAULT_MAX_SIZE)
           @data = {}
+          @mutex = Mutex.new
+          @max_size = max_size
+        end
+
+        def data
+          @mutex.synchronize { @data.dup }
         end
 
         def set(key, value)
-          @data[key.to_sym] = value
+          @mutex.synchronize do
+            evict_if_full
+            @data[key.to_sym] = value
+          end
         end
 
         def get(key)
-          @data[key.to_sym] if @data.key?(key.to_sym)
+          @mutex.synchronize do
+            @data[key.to_sym] if @data.key?(key.to_sym)
+          end
         end
 
         def has?(key)
-          @data.key?(key.to_sym)
+          @mutex.synchronize do
+            @data.key?(key.to_sym)
+          end
+        end
+
+        def size
+          @mutex.synchronize { @data.size }
+        end
+
+        def clear!
+          @mutex.synchronize { @data.clear }
+        end
+
+        private
+
+        def evict_if_full
+          return if @data.size < @max_size
+
+          # Remove the oldest entry (first inserted key)
+          oldest_key = @data.keys.first
+          @data.delete(oldest_key)
         end
       end
     end

data/lib/sastrawi/stemmer/cached_stemmer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'sastrawi/stemmer/filter/text_normalizer'
 
 module Sastrawi
@@ -10,9 +12,15 @@ module Sastrawi
         @delegated_stemmer = delegated_stemmer
       end
 
+      def clear_cache!
+        @cache.clear!
+      end
+
       def stem(text)
         normalized_text = Sastrawi::Stemmer::Filter::TextNormalizer.normalize_text(text)
 
+        return "" if normalized_text.empty?
+
         words = normalized_text.split(' ')
         stems = []
 

data/lib/sastrawi/stemmer/confix_stripping/precedence_adjustment_specification.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 ##
 # Confix Stripping Rule Precendence Adjustment Specification
 # Asian J. (2007) "Effective Techniques for Indonesian Text Retrieval" page 78-79

data/lib/sastrawi/stemmer/context/context.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'sastrawi/stemmer/confix_stripping/precedence_adjustment_specification'
 
 ##
@@ -205,11 +207,7 @@ module Sastrawi
             end
           end
 
-          @removals.each do |removal|
-            if removal.affix_type == 'DP'
-              @removals.delete(removal)
-            end
-          end
+          @removals = @removals.reject { |removal| removal.affix_type == 'DP' }
         end
       end
     end

data/lib/sastrawi/stemmer/context/removal.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Stemmer
     module Context

data/lib/sastrawi/stemmer/context/visitor/dont_stem_short_word.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Stemmer
     module Context

data/lib/sastrawi/stemmer/context/visitor/prefix_disambiguator.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Stemmer
     module Context

data/lib/sastrawi/stemmer/context/visitor/remove_derivational_suffix.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'sastrawi/stemmer/context/removal'
 
 ##

data/lib/sastrawi/stemmer/context/visitor/remove_inflectional_particle.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 ##
 # Remove inflectional particle
 # Asian J. (2007) "Effective Techniques for Indonesian Text Retrieval" page 60

data/lib/sastrawi/stemmer/context/visitor/remove_inflectional_possessive_pronoun.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 ##
 # Remove inflectional possessive pronoun
 # Asian J. (2007) "Effective Techniques for Indonesia Text Retrieval" page 60

data/lib/sastrawi/stemmer/context/visitor/remove_plain_prefix.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 ##
 # Remove plain prefix
 # Asian J. (2007) "Effective Techniques for Indonesian Text Retrieval" page 61

data/lib/sastrawi/stemmer/context/visitor/visitor_provider.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'sastrawi/stemmer/context/visitor/dont_stem_short_word'
 require 'sastrawi/stemmer/context/visitor/remove_inflectional_particle'
 require 'sastrawi/stemmer/context/visitor/remove_inflectional_possessive_pronoun'