sastrawi-ruby 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 293b0a6274835ba56e5df1e6c0a96b725e026ef703af178b7f791c3763b723c3
-  data.tar.gz: 2b74b53603072b2e4171abfc8117308dae795a78b2e0071fd73a7961e732808b
+  metadata.gz: b33ab69d9a6a019e376bb620152f76de882282d279fe141b93a0086b03165ac5
+  data.tar.gz: 6fbe94a6ee7443ce97c5a83042a8bd3bccf473a1b9bf578da9b6a3ad04250eca
 SHA512:
-  metadata.gz: dc74bcdbcb213b19c26e0dd6480077e69ae8dec455911b5004af662547fe0e7bd5af72943615025025d244b9f7532e1f6b062bf213b99ae8f0e96b7f8c8db73b
-  data.tar.gz: eb5807126530f67a26935fd85130206ee08da1a4817d0e1d1f7151b011dc6533f6ef16b4a41673a5c2af5ec85a776d0be84e77ee024da15a730d7b090786dab9
+  metadata.gz: c58d1702bb5ec1d2fa5a964c8aa15420e32946cb31a0c3227a8d8f27af737d9720ae1fd77b1a9d117d08fe969cad270f2a5e5ee7d54e3903a2a39c7f54ef793d
+  data.tar.gz: 5db74112b60b6c74fde9e1ad94488a998aa762f606b71a395ff99521ecea4db1e341e1bf1d6bafe8cf13a4d21ce55f7740085ad8f6546daedfdb3d97f2647e76

data/lib/sastrawi/dictionary/array_dictionary.rb

@@ -1,14 +1,23 @@
+# frozen_string_literal: true
+
+require 'set'
+
 module Sastrawi
   module Dictionary
     class ArrayDictionary
-      attr_reader :words
-
       def initialize(words = [])
-        @words = []
+        @words = Set.new
 
         add_words(words)
       end
 
+      ##
+      # Return the words as an Array (for backward compatibility)
+
+      def words
+        @words.to_a
+      end
+
       ##
       # Check whether a word is contained in the dictionary
 
@@ -20,7 +29,7 @@ module Sastrawi
       # Count how many words in the dictionary
 
       def count
-        @words.length
+        @words.size
       end
 
       ##
@@ -36,9 +45,13 @@ module Sastrawi
       # Add a word to the dictionary
 
       def add(word)
-        return if word.nil? || word.strip == ''
+        unless word.is_a?(String)
+          raise ArgumentError, "dictionary entries must be strings, got #{word.class}"
+        end
+
+        return if word.strip == ''
 
-        @words.push(word)
+        @words.add(word)
       end
 
       ##

data/lib/sastrawi/stemmer/cache/array_cache.rb

@@ -1,23 +1,58 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Stemmer
     module Cache
       class ArrayCache
-        attr_reader :data
+        DEFAULT_MAX_SIZE = 10_000
+
+        attr_reader :max_size
 
-        def initialize
+        def initialize(max_size: DEFAULT_MAX_SIZE)
           @data = {}
+          @mutex = Mutex.new
+          @max_size = max_size
+        end
+
+        def data
+          @mutex.synchronize { @data.dup }
         end
 
         def set(key, value)
-          @data[key.to_sym] = value
+          @mutex.synchronize do
+            evict_if_full
+            @data[key.to_sym] = value
+          end
         end
 
         def get(key)
-          @data[key.to_sym] if @data.key?(key.to_sym)
+          @mutex.synchronize do
+            @data[key.to_sym] if @data.key?(key.to_sym)
+          end
         end
 
         def has?(key)
-          @data.key?(key.to_sym)
+          @mutex.synchronize do
+            @data.key?(key.to_sym)
+          end
+        end
+
+        def size
+          @mutex.synchronize { @data.size }
+        end
+
+        def clear!
+          @mutex.synchronize { @data.clear }
+        end
+
+        private
+
+        def evict_if_full
+          return if @data.size < @max_size
+
+          # Remove the oldest entry (first inserted key)
+          oldest_key = @data.keys.first
+          @data.delete(oldest_key)
         end
       end
     end

data/lib/sastrawi/stemmer/cached_stemmer.rb

@@ -10,9 +10,15 @@ module Sastrawi
         @delegated_stemmer = delegated_stemmer
       end
 
+      def clear_cache!
+        @cache.clear!
+      end
+
       def stem(text)
         normalized_text = Sastrawi::Stemmer::Filter::TextNormalizer.normalize_text(text)
 
+        return "" if normalized_text.empty?
+
         words = normalized_text.split(' ')
         stems = []
 

data/lib/sastrawi/stemmer/context/context.rb

@@ -205,11 +205,7 @@ module Sastrawi
             end
           end
 
-          @removals.each do |removal|
-            if removal.affix_type == 'DP'
-              @removals.delete(removal)
-            end
-          end
+          @removals = @removals.reject { |removal| removal.affix_type == 'DP' }
         end
       end
     end

data/lib/sastrawi/stemmer/filter/text_normalizer.rb

@@ -1,8 +1,18 @@
+# frozen_string_literal: true
+
 module Sastrawi
   module Stemmer
     module Filter
       class TextNormalizer
         def self.normalize_text(text)
+          return "" if text.nil?
+
+          unless text.is_a?(String)
+            raise ArgumentError, "expected a String, got #{text.class}"
+          end
+
+          return "" if text.empty?
+
           lowercase_text = text.downcase
           replaced_text = lowercase_text.gsub(/[^a-z0-9 -]/im, ' ')
           replaced_text = replaced_text.gsub(/( +)/im, ' ')

data/lib/sastrawi/stemmer/stemmer.rb

@@ -24,6 +24,8 @@ module Sastrawi
       def stem(text)
         normalized_text = Sastrawi::Stemmer::Filter::TextNormalizer.normalize_text(text)
 
+        return "" if normalized_text.empty?
+
         words = normalized_text.split(' ')
         stems = []
 

data/lib/sastrawi/stop_word_remover/stop_word_remover.rb

@@ -11,6 +11,8 @@ module Sastrawi
       # Remove stop words
 
       def remove(text)
+        return "" if text.nil? || text.empty?
+
         words = text.split(' ')
         stop_words = []
 

data/lib/sastrawi/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Sastrawi
-  VERSION = "0.2.0"
+  VERSION = "0.2.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sastrawi-ruby
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Johannes Dwi Cahyo