sastrawi 0.1.0.pre → 0.1.4

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule8.rb

@@ -10,7 +10,7 @@ module Sastrawi
 
             return if matches[0] == 'r' || /^er(.*)$/.match(matches[1])
 
-            return matches[0] << matches[1]
+            return "#{matches[0]}#{matches[1]}"
           end
         end
       end

data/lib/sastrawi/morphology/disambiguator/disambiguator_prefix_rule9.rb

@@ -3,14 +3,14 @@ module Sastrawi
     module Disambiguator
       class DisambiguatorPrefixRule9
         def disambiguate(word)
-          contains = /^te([bcdfghjklmnpqrstvwxyz])er(([bcdfghjklmnpqrstvwxyz]).*)$/.match(word)
+          contains = /^te([bcdfghjklmnpqrstvwxyz])er([bcdfghjklmnpqrstvwxyz])(.*)$/.match(word)
 
           if contains
             matches = contains.captures
 
             return if matches[0] == 'r'
 
-            return matches[0] << 'er' << matches[1] << matches[2]
+            return "#{matches[0]}er#{matches[1]}#{matches[2]}"
           end
         end
       end

data/lib/sastrawi/morphology/invalid_affix_pair_specification.rb

@@ -1,3 +1,7 @@
+##
+# Asian J. (2007) "Effective Techniques for Indonesian Text Retrieval". page 26
+# http://researchbank.rmit.edu.au/eserv/rmit:6312/Asian.pdf
+
 module Sastrawi
   module Morphology
     class InvalidAffixPairSpecification

data/lib/sastrawi/stemmer/cache/array_cache.rb

@@ -2,7 +2,7 @@ module Sastrawi
   module Stemmer
     module Cache
       class ArrayCache
-        attr_accessor :data
+        attr_reader :data
 
         def initialize
           @data = {}
@@ -13,7 +13,7 @@ module Sastrawi
         end
 
         def get(key)
-          return @data[key.to_sym] if @data.key?(key.to_sym)
+          @data[key.to_sym] if @data.key?(key.to_sym)
         end
 
         def has?(key)

data/lib/sastrawi/stemmer/cached_stemmer.rb

@@ -3,7 +3,7 @@ require 'sastrawi/stemmer/filter/text_normalizer'
 module Sastrawi
   module Stemmer
     class CachedStemmer
-      attr_accessor :cache, :delegated_stemmer
+      attr_reader :cache, :delegated_stemmer
 
       def initialize(cache, delegated_stemmer)
         @cache = cache

data/lib/sastrawi/stemmer/confix_stripping/precedence_adjustment_specification.rb

@@ -1,3 +1,8 @@
+##
+# Confix Stripping Rule Precendence Adjustment Specification
+# Asian J. (2007) "Effective Techniques for Indonesian Text Retrieval" page 78-79
+# http://researchbank.rmit.edu.au/eserv/rmit:6312/Asian.pdf
+
 module Sastrawi
   module Stemmer
     module ConfixStripping

data/lib/sastrawi/stemmer/context/context.rb

@@ -1,10 +1,15 @@
 require 'sastrawi/stemmer/confix_stripping/precedence_adjustment_specification'
 
+##
+# Stemming context using Nazief and Adriani, Confix Stripping (CS),
+# Enhanced Confix Stripping (ECS), and Improved (ECS)
+
 module Sastrawi
   module Stemmer
     module Context
       class Context
-        attr_accessor :original_word, :current_word, :dictionary, :visitor_provider, :process_is_stopped, :removals, :visitors, :suffix_visitors, :prefix_visitors, :result
+        attr_reader :original_word, :dictionary, :visitor_provider, :visitors, :suffix_visitors, :prefix_visitors
+        attr_accessor :current_word, :process_is_stopped, :removals, :result
 
         def initialize(original_word, dictionary, visitor_provider)
             @original_word = original_word
@@ -14,10 +19,10 @@ module Sastrawi
 
             @process_is_stopped = false
             @removals = []
-            @visitors = []
-            @suffix_visitors = []
-            @prefix_visitors = []
-            @result = ''
+            @visitors = nil
+            @suffix_visitors = nil
+            @prefix_visitors = nil
+            @result = nil
 
             init_visitors
         end
@@ -36,6 +41,9 @@ module Sastrawi
           @removals.push(removal)
         end
 
+        ##
+        # Execute stemming process
+
         def execute
           start_stemming_process
 
@@ -55,6 +63,10 @@ module Sastrawi
 
           cs_precendence_adjustment_specification = Sastrawi::Stemmer::ConfixStripping::PrecedenceAdjustmentSpecification.new
 
+          ##
+          # Confix stripping
+          # try to remove prefix before suffix if the specification is met
+
           if cs_precendence_adjustment_specification.satisfied_by?(@original_word)
             remove_prefixes
             return if @dictionary.contains?(@current_word)
@@ -77,6 +89,9 @@ module Sastrawi
           loop_last_return
         end
 
+        ##
+        # ECS loop last return
+
         def loop_last_return
           restore_prefix
 
@@ -88,12 +103,12 @@ module Sastrawi
             next unless suffix_removal?(reverse_removal)
 
             if reverse_removal.removed_part == 'kan'
-              @current_word = reverse_removal.result << 'k'
+              @current_word = "#{reverse_removal.result}k"
 
               remove_prefixes
               return if @dictionary.contains?(@current_word)
 
-              @current_word = reverse_removal.result << 'kan'
+              @current_word = "#{reverse_removal.result}kan"
             else
               @current_word = reverse_removal.subject
             end
@@ -146,10 +161,16 @@ module Sastrawi
           end
         end
 
+        ##
+        # Check whether the removed part is a suffix
+
         def suffix_removal?(removal)
           removal.affix_type == 'DS' || removal.affix_type == 'PP' || removal.affix_type == 'P'
         end
 
+        ##
+        # Restore prefix to proceed with ECS loop last return
+
         def restore_prefix
           @removals.each do |removal|
             if removal.affix_type == 'DP'

data/lib/sastrawi/stemmer/context/removal.rb

@@ -2,7 +2,7 @@ module Sastrawi
   module Stemmer
     module Context
       class Removal
-        attr_accessor :visitor, :subject, :result, :removed_part, :affix_type
+        attr_reader :visitor, :subject, :result, :removed_part, :affix_type
 
         def initialize(visitor, subject, result, removed_part, affix_type)
           @visitor = visitor

data/lib/sastrawi/stemmer/context/visitor/prefix_disambiguator.rb

@@ -3,7 +3,7 @@ module Sastrawi
     module Context
       module Visitor
         class PrefixDisambiguator
-          attr_accessor :disambiguators
+          attr_reader :disambiguators
 
           def initialize(disambiguators = [])
             @disambiguators = []
@@ -22,7 +22,7 @@ module Sastrawi
 
             return if result.nil?
 
-            removed_part = context.current_word.sub(result, '')
+            removed_part = context.current_word.sub(/#{Regexp.quote(result)}/, '')
 
             removal = Removal.new(self, context.current_word, result, removed_part, 'DP')
 

data/lib/sastrawi/stemmer/context/visitor/remove_derivational_suffix.rb

@@ -1,5 +1,10 @@
 require 'sastrawi/stemmer/context/removal'
 
+##
+# Remove derivational suffix
+# Asian J. (2007) "Effective Techniques for Indonesian Text Retrieval" page 61
+# http://researchbank.rmit.edu.au/eserv/rmit:6312/Asian.pdf
+
 module Sastrawi
   module Stemmer
     module Context
@@ -9,7 +14,7 @@ module Sastrawi
             result = remove_suffix(context.current_word)
 
             if result != context.current_word
-              removed_part = context.current_word.sub(result, '')
+              removed_part = context.current_word.sub(/#{Regexp.quote(result)}/, '')
 
               removal = Sastrawi::Stemmer::Context::Removal.new(self, context.current_word, result, removed_part, 'DS')
 
@@ -18,6 +23,10 @@ module Sastrawi
             end
           end
 
+          ##
+          # Original rule: i|kan|an
+          # Added the adopted foreign suffix rule: is|isme|isasi
+
           def remove_suffix(word)
             word.sub(/(is|isme|isasi|i|kan|an)$/, '')
           end

data/lib/sastrawi/stemmer/context/visitor/remove_inflectional_particle.rb

@@ -1,3 +1,8 @@
+##
+# Remove inflectional particle
+# Asian J. (2007) "Effective Techniques for Indonesian Text Retrieval" page 60
+# http://researchbank.rmit.edu.au/eserv/rmit:6312/Asian.pdf
+
 module Sastrawi
   module Stemmer
     module Context
@@ -7,7 +12,7 @@ module Sastrawi
             result = remove(context.current_word)
 
             if result != context.current_word
-              removed_part = context.current_word.sub(result, '')
+              removed_part = context.current_word.sub(/#{Regexp.quote(result)}/, '')
 
               removal = Removal.new(self, context.current_word, result, removed_part, 'P')
 
@@ -16,6 +21,9 @@ module Sastrawi
             end
           end
 
+          ##
+          # Remove inflectional particle: lah|kah|tah|pun
+
           def remove(word)
             word.sub(/-*(lah|kah|tah|pun)$/, '')
           end

data/lib/sastrawi/stemmer/context/visitor/remove_inflectional_possessive_pronoun.rb

@@ -1,3 +1,8 @@
+##
+# Remove inflectional possessive pronoun
+# Asian J. (2007) "Effective Techniques for Indonesia Text Retrieval" page 60
+# http://researchbank.rmit.edu.au/eserv/rmit:6312/Asian.pdf
+
 module Sastrawi
   module Stemmer
     module Context
@@ -7,7 +12,7 @@ module Sastrawi
             result = remove(context.current_word)
 
             if result != context.current_word
-              removed_part = context.current_word.sub(result, '')
+              removed_part = context.current_word.sub(/#{Regexp.quote(result)}/, '')
 
               removal = Removal.new(self, context.current_word, result, removed_part, 'PP')
 
@@ -16,6 +21,9 @@ module Sastrawi
             end
           end
 
+          ##
+          # Remove inflectional possessive pronoun: ku|mu|nya|
+
           def remove(word)
             word.sub(/-*(ku|mu|nya)$/, '')
           end

data/lib/sastrawi/stemmer/context/visitor/remove_plain_prefix.rb

@@ -1,3 +1,8 @@
+##
+# Remove plain prefix
+# Asian J. (2007) "Effective Techniques for Indonesian Text Retrieval" page 61
+# http://researchbank.rmit.edu.au/eserv/rmit:6312/Asian.pdf
+
 module Sastrawi
   module Stemmer
     module Context
@@ -7,7 +12,7 @@ module Sastrawi
             result = remove(context.current_word)
 
             if result != context.current_word
-              removed_part = context.current_word.sub(result, '')
+              removed_part = context.current_word.sub(/#{Regexp.quote(result)}/, '')
 
               removal = Removal.new(self, context.current_word, result, removed_part, 'DP')
 
@@ -16,6 +21,9 @@ module Sastrawi
             end
           end
 
+          ##
+          # Remove plain prefix: di|ke|se
+
           def remove(word)
             word.sub(/^(di|ke|se)/, '')
           end

data/lib/sastrawi/stemmer/context/visitor/visitor_provider.rb

@@ -69,7 +69,7 @@ module Sastrawi
     module Context
       module Visitor
         class VisitorProvider
-          attr_accessor :visitors, :suffix_visitors, :prefix_visitors
+          attr_reader :visitors, :suffix_visitors, :prefix_visitors
 
           def initialize
             @visitors = []

data/lib/sastrawi/stemmer/stemmer.rb

@@ -1,17 +1,26 @@
 require 'sastrawi/stemmer/context/context'
+
 require 'sastrawi/stemmer/context/visitor/visitor_provider'
+
 require 'sastrawi/stemmer/filter/text_normalizer'
 
+##
+# Indonesian Stemmer
+# Nazief & Adriani, CS Stemmer, ECS Stemmer, Improved ECS
+
 module Sastrawi
   module Stemmer
     class Stemmer
-      attr_accessor :dictionary, :visitor_provider
+      attr_reader :dictionary, :visitor_provider
 
       def initialize(dictionary)
         @dictionary = dictionary
         @visitor_provider = Sastrawi::Stemmer::Context::Visitor::VisitorProvider.new
       end
 
+      ##
+      # Stem a string to its base form
+
       def stem(text)
         normalized_text = Sastrawi::Stemmer::Filter::TextNormalizer.normalize_text(text)
 
@@ -25,6 +34,9 @@ module Sastrawi
         stems.join(' ')
       end
 
+      ##
+      # Stem a word to its base form
+
       def stem_word(word)
         if plural?(word)
           stem_plural_word(word)
@@ -36,35 +48,36 @@ module Sastrawi
       def plural?(word)
         matches = /^(.*)-(ku|mu|nya|lah|kah|tah|pun)$/.match(word)
 
-        if matches
-          true
-        else
-          false
-        end
+        return matches[1].include?('-') if matches
+
+        return word.include?('-')
       end
 
+      ##
+      # Stem a plural word to its base form
+      # Asian J. (2007) "Effective Techniques for Indonesian Text Retrieval"
+      # page 76-77
+
       def stem_plural_word(word)
         first_match = /^(.*)-(.*)$/.match(word)
 
-        unless first_match
-          return word
-        end
-
-        words = [first_match.captures[0], first_match.captures[1]]
+        return word unless first_match
 
+        words = [first_match[1], first_match[2]]
         suffix = words[1]
-        suffixes = ['ku', 'mu', 'nya', 'lah', 'kah', 'tah', 'pun']
+        suffixes = %w[ku mu nya lah kah tah pun]
         second_match = /^(.*)-(.*)$/.match(words[0])
 
         if suffixes.include?(suffix) && second_match
-          words[1] = words[1] + '-' + suffix
+          words[0] = second_match[1]
+          words[1] = "#{second_match[2]}-#{suffix}"
         end
 
         root_first_word = stem_singular_word(words[0])
         root_second_word = stem_singular_word(words[1])
 
-        unless @dictionary.contains?(words[1]) && root_second_word == words[1]
-          root_second_word = stem_singular_word('me' + words[1])
+        if !@dictionary.contains?(words[1]) && root_second_word == words[1]
+          root_second_word = stem_singular_word("me#{words[1]}")
         end
 
         if root_first_word == root_second_word
@@ -74,6 +87,9 @@ module Sastrawi
         end
       end
 
+      ##
+      # Stem a singular word to its base form
+
       def stem_singular_word(word)
         context = Sastrawi::Stemmer::Context::Context.new(word, @dictionary, @visitor_provider)
         context.execute

data/lib/sastrawi/stemmer/stemmer_factory.rb

@@ -1,8 +1,12 @@
 require 'sastrawi/dictionary/array_dictionary'
+
 require 'sastrawi/stemmer/cached_stemmer'
 require 'sastrawi/stemmer/stemmer'
+
 require 'sastrawi/stemmer/cache/array_cache'
 
+##
+# Stemmer factory helps creating a pre-configured stemmer
 
 module Sastrawi
   module Stemmer
@@ -29,7 +33,7 @@ module Sastrawi
 
       def get_words_from_file
         root_directory = File.expand_path('../../../..', __FILE__)
-        dictionary_file_path = File.join(root_directory, 'data/kata-dasar.txt')
+        dictionary_file_path = File.join(root_directory, 'data/base-word.txt')
 
         dictionary_content = []
         File.open(dictionary_file_path, 'r') do |file|