sastrawi-ruby 0.2.0

data/lib/sastrawi/stemmer/context/context.rb

@@ -0,0 +1,217 @@
+require 'sastrawi/stemmer/confix_stripping/precedence_adjustment_specification'
+
+##
+# Stemming context using Nazief and Adriani, Confix Stripping (CS),
+# Enhanced Confix Stripping (ECS), and Improved (ECS)
+
+module Sastrawi
+  module Stemmer
+    module Context
+      class Context
+        attr_reader :original_word, :dictionary, :visitor_provider, :visitors, :suffix_visitors, :prefix_visitors
+        attr_accessor :current_word, :process_is_stopped, :removals, :result
+
+        def initialize(original_word, dictionary, visitor_provider)
+            @original_word = original_word
+            @current_word = original_word
+            @dictionary = dictionary
+            @visitor_provider = visitor_provider
+
+            @process_is_stopped = false
+            @removals = []
+            @visitors = nil
+            @suffix_visitors = nil
+            @prefix_visitors = nil
+            @result = nil
+
+            init_visitors
+        end
+
+        def init_visitors
+          @visitors = @visitor_provider.visitors
+          @suffix_visitors = @visitor_provider.suffix_visitors
+          @prefix_visitors = @visitor_provider.prefix_visitors
+        end
+
+        def stop_process
+          @process_is_stopped = true
+        end
+
+        def add_removal(removal)
+          @removals.push(removal)
+        end
+
+        ##
+        # Execute stemming process
+
+        def execute
+          start_stemming_process
+
+          if @dictionary.contains?(@current_word)
+            @result = @current_word
+          else
+            @result = @original_word
+          end
+        end
+
+        def start_stemming_process
+          return if @dictionary.contains?(@current_word)
+
+          accept_visitors(@visitors)
+
+          return if @dictionary.contains?(@current_word)
+
+          cs_precendence_adjustment_specification = Sastrawi::Stemmer::ConfixStripping::PrecedenceAdjustmentSpecification.new
+
+          ##
+          # Confix stripping
+          # try to remove prefix before suffix if the specification is met
+
+          if cs_precendence_adjustment_specification.satisfied_by?(@original_word)
+            # Interleave prefix and suffix removal in CS path:
+            # After each prefix removal round, try suffix removal to check
+            # if the combination yields a dictionary word.
+            cs_found = false
+            3.times do
+              prev_removals_len = @removals.length
+              accept_prefix_visitors(@prefix_visitors)
+
+              break if @removals.length == prev_removals_len # no prefix removed
+
+              if @dictionary.contains?(@current_word)
+                cs_found = true
+                break
+              end
+
+              # Try suffix removal from this intermediate state
+              saved_word = @current_word
+              saved_removals = @removals.dup
+              remove_suffixes
+
+              if @dictionary.contains?(@current_word)
+                cs_found = true
+                break
+              end
+
+              # Suffix didn't help, restore and try more prefix iterations
+              @current_word = saved_word
+              @removals = saved_removals
+            end
+
+            if cs_found
+              return
+            else
+              @current_word = @original_word
+              @removals = []
+            end
+          end
+
+          remove_suffixes
+          return if @dictionary.contains?(@current_word)
+
+          remove_prefixes
+          return if @dictionary.contains?(@current_word)
+
+          loop_last_return
+        end
+
+        ##
+        # ECS loop last return
+
+        def loop_last_return
+          restore_prefix
+
+          removals = @removals
+          reversed_removals = removals.reverse
+          current_word = @current_word
+
+          reversed_removals.each do |reverse_removal|
+            next unless suffix_removal?(reverse_removal)
+
+            if reverse_removal.removed_part == 'kan'
+              @current_word = "#{reverse_removal.result}k"
+
+              remove_prefixes
+              return if @dictionary.contains?(@current_word)
+
+              @current_word = "#{reverse_removal.result}kan"
+            else
+              @current_word = reverse_removal.subject
+            end
+
+            remove_prefixes
+            return if @dictionary.contains?(@current_word)
+
+            @removals = removals
+            @current_word = current_word
+          end
+        end
+
+        def remove_prefixes
+          3.times do
+            accept_prefix_visitors(@prefix_visitors)
+
+            return if @dictionary.contains?(@current_word)
+          end
+        end
+
+        def remove_suffixes
+          accept_visitors(@suffix_visitors)
+        end
+
+        def accept(visitor)
+          visitor.visit(self)
+        end
+
+        def accept_visitors(visitors)
+          visitors.each do |visitor|
+            accept(visitor)
+
+            return @current_word if @dictionary.contains?(@current_word)
+
+            return @current_word if @process_is_stopped
+          end
+        end
+
+        def accept_prefix_visitors(visitors)
+          removal_length = @removals.length
+
+          visitors.each do |visitor|
+            accept(visitor)
+
+            return @current_word if @dictionary.contains?(@current_word)
+
+            return @current_word if @process_is_stopped
+
+            return if @removals.length > removal_length
+          end
+        end
+
+        ##
+        # Check whether the removed part is a suffix
+
+        def suffix_removal?(removal)
+          removal.affix_type == 'DS' || removal.affix_type == 'PP' || removal.affix_type == 'P'
+        end
+
+        ##
+        # Restore prefix to proceed with ECS loop last return
+
+        def restore_prefix
+          @removals.each do |removal|
+            if removal.affix_type == 'DP'
+              @current_word = removal.subject
+              break
+            end
+          end
+
+          @removals.each do |removal|
+            if removal.affix_type == 'DP'
+              @removals.delete(removal)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sastrawi/stemmer/context/removal.rb

@@ -0,0 +1,17 @@
+module Sastrawi
+  module Stemmer
+    module Context
+      class Removal
+        attr_reader :visitor, :subject, :result, :removed_part, :affix_type
+
+        def initialize(visitor, subject, result, removed_part, affix_type)
+          @visitor = visitor
+          @subject = subject
+          @result = result
+          @removed_part = removed_part
+          @affix_type = affix_type
+        end
+      end
+    end
+  end
+end

data/lib/sastrawi/stemmer/context/visitor/dont_stem_short_word.rb

@@ -0,0 +1,17 @@
+module Sastrawi
+  module Stemmer
+    module Context
+      module Visitor
+        class DontStemShortWord
+          def visit(context)
+            context.stop_process if short_word?(context.current_word)
+          end
+
+          def short_word?(word)
+            word.length <= 3
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sastrawi/stemmer/context/visitor/prefix_disambiguator.rb

@@ -0,0 +1,54 @@
+module Sastrawi
+  module Stemmer
+    module Context
+      module Visitor
+        class PrefixDisambiguator
+          attr_reader :disambiguators
+
+          def initialize(disambiguators = [])
+            @disambiguators = []
+
+            add_disambiguators(disambiguators)
+          end
+
+          def visit(context)
+            first_result = nil
+            dict_result = nil
+
+            @disambiguators.each do |disambiguator|
+              result = disambiguator.disambiguate(context.current_word)
+              next if result.nil?
+
+              first_result ||= result
+
+              if context.dictionary.contains?(result)
+                dict_result = result
+                break
+              end
+            end
+
+            result = dict_result || first_result
+            return if result.nil?
+
+            removed_part = context.current_word.sub(/#{Regexp.quote(result)}/, '')
+
+            removal = Removal.new(self, context.current_word, result, removed_part, 'DP')
+
+            context.add_removal(removal)
+            context.current_word = result
+          end
+
+          def add_disambiguators(disambiguators)
+            disambiguators.each do |disambiguator|
+              add_disambiguator(disambiguator)
+            end
+          end
+
+          def add_disambiguator(disambiguator)
+            @disambiguators.push(disambiguator)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sastrawi/stemmer/context/visitor/remove_derivational_suffix.rb

@@ -0,0 +1,37 @@
+require 'sastrawi/stemmer/context/removal'
+
+##
+# Remove derivational suffix
+# Asian J. (2007) "Effective Techniques for Indonesian Text Retrieval" page 61
+# http://researchbank.rmit.edu.au/eserv/rmit:6312/Asian.pdf
+
+module Sastrawi
+  module Stemmer
+    module Context
+      module Visitor
+        class RemoveDerivationalSuffix
+          def visit(context)
+            result = remove_suffix(context.current_word)
+
+            if result != context.current_word
+              removed_part = context.current_word.sub(/#{Regexp.quote(result)}/, '')
+
+              removal = Sastrawi::Stemmer::Context::Removal.new(self, context.current_word, result, removed_part, 'DS')
+
+              context.add_removal(removal)
+              context.current_word = result
+            end
+          end
+
+          ##
+          # Original rule: i|kan|an
+          # Added the adopted foreign suffix rule: is|isme|isasi
+
+          def remove_suffix(word)
+            word.sub(/(is|isme|isasi|i|kan|an)$/, '')
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sastrawi/stemmer/context/visitor/remove_inflectional_particle.rb

@@ -0,0 +1,34 @@
+##
+# Remove inflectional particle
+# Asian J. (2007) "Effective Techniques for Indonesian Text Retrieval" page 60
+# http://researchbank.rmit.edu.au/eserv/rmit:6312/Asian.pdf
+
+module Sastrawi
+  module Stemmer
+    module Context
+      module Visitor
+        class RemoveInflectionalParticle
+          def visit(context)
+            result = remove(context.current_word)
+
+            if result != context.current_word
+              removed_part = context.current_word.sub(/#{Regexp.quote(result)}/, '')
+
+              removal = Removal.new(self, context.current_word, result, removed_part, 'P')
+
+              context.add_removal(removal)
+              context.current_word = result
+            end
+          end
+
+          ##
+          # Remove inflectional particle: lah|kah|tah|pun
+
+          def remove(word)
+            word.sub(/-*(lah|kah|tah|pun)$/, '')
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sastrawi/stemmer/context/visitor/remove_inflectional_possessive_pronoun.rb

@@ -0,0 +1,34 @@
+##
+# Remove inflectional possessive pronoun
+# Asian J. (2007) "Effective Techniques for Indonesia Text Retrieval" page 60
+# http://researchbank.rmit.edu.au/eserv/rmit:6312/Asian.pdf
+
+module Sastrawi
+  module Stemmer
+    module Context
+      module Visitor
+        class RemoveInflectionalPossessivePronoun
+          def visit(context)
+            result = remove(context.current_word)
+
+            if result != context.current_word
+              removed_part = context.current_word.sub(/#{Regexp.quote(result)}/, '')
+
+              removal = Removal.new(self, context.current_word, result, removed_part, 'PP')
+
+              context.add_removal(removal)
+              context.current_word = result
+            end
+          end
+
+          ##
+          # Remove inflectional possessive pronoun: ku|mu|nya|
+
+          def remove(word)
+            word.sub(/-*(ku|mu|nya)$/, '')
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sastrawi/stemmer/context/visitor/remove_plain_prefix.rb

@@ -0,0 +1,34 @@
+##
+# Remove plain prefix
+# Asian J. (2007) "Effective Techniques for Indonesian Text Retrieval" page 61
+# http://researchbank.rmit.edu.au/eserv/rmit:6312/Asian.pdf
+
+module Sastrawi
+  module Stemmer
+    module Context
+      module Visitor
+        class RemovePlainPrefix
+          def visit(context)
+            result = remove(context.current_word)
+
+            if result != context.current_word
+              removed_part = context.current_word.sub(/#{Regexp.quote(result)}/, '')
+
+              removal = Removal.new(self, context.current_word, result, removed_part, 'DP')
+
+              context.add_removal(removal)
+              context.current_word = result
+            end
+          end
+
+          ##
+          # Remove plain prefix: di|ke|se
+
+          def remove(word)
+            word.sub(/^(di|ke|se)/, '')
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sastrawi/stemmer/context/visitor/visitor_provider.rb

@@ -0,0 +1,157 @@
+require 'sastrawi/stemmer/context/visitor/dont_stem_short_word'
+require 'sastrawi/stemmer/context/visitor/remove_inflectional_particle'
+require 'sastrawi/stemmer/context/visitor/remove_inflectional_possessive_pronoun'
+require 'sastrawi/stemmer/context/visitor/remove_derivational_suffix'
+require 'sastrawi/stemmer/context/visitor/remove_plain_prefix'
+require 'sastrawi/stemmer/context/visitor/prefix_disambiguator'
+
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule1a'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule1b'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule2'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule3'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule4'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule5'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule6a'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule6b'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule7'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule8'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule9'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule10'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule11'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule12'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule13a'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule13b'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule14'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule15a'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule15b'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule16'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule17a'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule17b'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule17c'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule17d'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule18a'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule18b'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule19'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule20'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule21a'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule21b'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule23'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule24'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule25'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule26a'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule26b'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule27'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule28a'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule28b'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule29'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule30a'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule30b'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule30c'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule31a'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule31b'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule32'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule34'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule35'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule36'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule37a'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule37b'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule38a'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule38b'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule39a'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule39b'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule40a'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule40b'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule41'
+require 'sastrawi/morphology/disambiguator/disambiguator_prefix_rule42'
+
+module Sastrawi
+  module Stemmer
+    module Context
+      module Visitor
+        class VisitorProvider
+          attr_reader :visitors, :suffix_visitors, :prefix_visitors
+
+          def initialize
+            @visitors = []
+            @suffix_visitors = []
+            @prefix_visitors = []
+
+            init_visitors
+          end
+
+          def init_visitors
+            @visitors.push(Sastrawi::Stemmer::Context::Visitor::DontStemShortWord.new)
+
+            @suffix_visitors.push(Sastrawi::Stemmer::Context::Visitor::RemoveInflectionalParticle.new)
+            @suffix_visitors.push(Sastrawi::Stemmer::Context::Visitor::RemoveInflectionalPossessivePronoun.new)
+            @suffix_visitors.push(Sastrawi::Stemmer::Context::Visitor::RemoveDerivationalSuffix.new)
+
+            @prefix_visitors.push(Sastrawi::Stemmer::Context::Visitor::RemovePlainPrefix.new)
+            @prefix_visitors.push(Sastrawi::Stemmer::Context::Visitor::PrefixDisambiguator.new([Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule1a.new, Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule1b.new]))
+            @prefix_visitors.push(Sastrawi::Stemmer::Context::Visitor::PrefixDisambiguator.new([Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule2.new]))
+            @prefix_visitors.push(Sastrawi::Stemmer::Context::Visitor::PrefixDisambiguator.new([Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule3.new]))
+            @prefix_visitors.push(Sastrawi::Stemmer::Context::Visitor::PrefixDisambiguator.new([Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule4.new]))
+            @prefix_visitors.push(Sastrawi::Stemmer::Context::Visitor::PrefixDisambiguator.new([Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule5.new]))
+            @prefix_visitors.push(Sastrawi::Stemmer::Context::Visitor::PrefixDisambiguator.new([Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule6a.new, Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule6b.new]))
+
+            @prefix_visitors.push(Sastrawi::Stemmer::Context::Visitor::PrefixDisambiguator.new([Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule7.new]))
+            @prefix_visitors.push(Sastrawi::Stemmer::Context::Visitor::PrefixDisambiguator.new([Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule8.new]))
+            @prefix_visitors.push(Sastrawi::Stemmer::Context::Visitor::PrefixDisambiguator.new([Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule9.new]))
+            @prefix_visitors.push(Sastrawi::Stemmer::Context::Visitor::PrefixDisambiguator.new([Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule10.new]))
+            @prefix_visitors.push(Sastrawi::Stemmer::Context::Visitor::PrefixDisambiguator.new([Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule11.new]))
+            @prefix_visitors.push(Sastrawi::Stemmer::Context::Visitor::PrefixDisambiguator.new([Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule12.new]))
+            @prefix_visitors.push(Sastrawi::Stemmer::Context::Visitor::PrefixDisambiguator.new([Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule13a.new, Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule13b.new]))
+
+            @prefix_visitors.push(Sastrawi::Stemmer::Context::Visitor::PrefixDisambiguator.new([Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule14.new]))
+            @prefix_visitors.push(Sastrawi::Stemmer::Context::Visitor::PrefixDisambiguator.new([Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule15a.new, Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule15b.new]))
+            @prefix_visitors.push(Sastrawi::Stemmer::Context::Visitor::PrefixDisambiguator.new([Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule16.new]))
+
+            @prefix_visitors.push(Sastrawi::Stemmer::Context::Visitor::PrefixDisambiguator.new([
+              Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule17a.new, Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule17b.new,
+              Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule17c.new, Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule17d.new
+            ]))
+
+            @prefix_visitors.push(Sastrawi::Stemmer::Context::Visitor::PrefixDisambiguator.new([Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule18a.new, Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule18b.new]))
+            @prefix_visitors.push(Sastrawi::Stemmer::Context::Visitor::PrefixDisambiguator.new([Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule19.new]))
+            @prefix_visitors.push(Sastrawi::Stemmer::Context::Visitor::PrefixDisambiguator.new([Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule20.new]))
+            @prefix_visitors.push(Sastrawi::Stemmer::Context::Visitor::PrefixDisambiguator.new([Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule21a.new, Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule21b.new]))
+            @prefix_visitors.push(Sastrawi::Stemmer::Context::Visitor::PrefixDisambiguator.new([Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule23.new]))
+            @prefix_visitors.push(Sastrawi::Stemmer::Context::Visitor::PrefixDisambiguator.new([Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule24.new]))
+            @prefix_visitors.push(Sastrawi::Stemmer::Context::Visitor::PrefixDisambiguator.new([Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule25.new]))
+
+            @prefix_visitors.push(Sastrawi::Stemmer::Context::Visitor::PrefixDisambiguator.new([Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule26a.new, Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule26b.new]))
+
+            @prefix_visitors.push(Sastrawi::Stemmer::Context::Visitor::PrefixDisambiguator.new([Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule27.new]))
+
+            @prefix_visitors.push(Sastrawi::Stemmer::Context::Visitor::PrefixDisambiguator.new([Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule28a.new, Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule28b.new]))
+
+            @prefix_visitors.push(Sastrawi::Stemmer::Context::Visitor::PrefixDisambiguator.new([Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule29.new]))
+
+            @prefix_visitors.push(Sastrawi::Stemmer::Context::Visitor::PrefixDisambiguator.new([
+              Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule30a.new, Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule30b.new,
+              Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule30c.new
+            ]))
+
+            @prefix_visitors.push(Sastrawi::Stemmer::Context::Visitor::PrefixDisambiguator.new([Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule31a.new, Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule31b.new]))
+            @prefix_visitors.push(Sastrawi::Stemmer::Context::Visitor::PrefixDisambiguator.new([Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule32.new]))
+            @prefix_visitors.push(Sastrawi::Stemmer::Context::Visitor::PrefixDisambiguator.new([Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule34.new]))
+
+            @prefix_visitors.push(Sastrawi::Stemmer::Context::Visitor::PrefixDisambiguator.new([Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule35.new]))
+            @prefix_visitors.push(Sastrawi::Stemmer::Context::Visitor::PrefixDisambiguator.new([Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule36.new]))
+
+            @prefix_visitors.push(Sastrawi::Stemmer::Context::Visitor::PrefixDisambiguator.new([Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule37a.new, Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule37b.new]))
+
+            @prefix_visitors.push(Sastrawi::Stemmer::Context::Visitor::PrefixDisambiguator.new([Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule38a.new, Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule38b.new]))
+
+            @prefix_visitors.push(Sastrawi::Stemmer::Context::Visitor::PrefixDisambiguator.new([Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule39a.new, Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule39b.new]))
+
+            @prefix_visitors.push(Sastrawi::Stemmer::Context::Visitor::PrefixDisambiguator.new([Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule40a.new, Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule40b.new]))
+
+            @prefix_visitors.push(Sastrawi::Stemmer::Context::Visitor::PrefixDisambiguator.new([Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule41.new]))
+            @prefix_visitors.push(Sastrawi::Stemmer::Context::Visitor::PrefixDisambiguator.new([Sastrawi::Morphology::Disambiguator::DisambiguatorPrefixRule42.new]))
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sastrawi/stemmer/filter/text_normalizer.rb

@@ -0,0 +1,15 @@
+module Sastrawi
+  module Stemmer
+    module Filter
+      class TextNormalizer
+        def self.normalize_text(text)
+          lowercase_text = text.downcase
+          replaced_text = lowercase_text.gsub(/[^a-z0-9 -]/im, ' ')
+          replaced_text = replaced_text.gsub(/( +)/im, ' ')
+
+          replaced_text.strip
+        end
+      end
+    end
+  end
+end