indonesian_stemmer 0.1.1 → 0.2.0

data/lib/indonesian_stemmer/morphological_utility.rb

@@ -1,4 +1,5 @@
 require "indonesian_stemmer/stemmer_utility"
+require "indonesian_stemmer/irregular_words"
 
 module IndonesianStemmer
 
@@ -15,57 +16,6 @@ module IndonesianStemmer
   SUFFIX_CHARACTERS                           = %w( kan an i )
   WITH_VOWEL_SUBSTITUTION_PREFIX_CHARACTERS   = %w( meny peny men pen )
 
-  IRREGULARS_FOR_WORDS_BEGINS_WITH_K  = %w(
-    aget alah andung ata ejar eluar embali empis emuka ena enal encang endali ering
-    erja erut etahui etik ibar irim uasai uliti umpul unci unjung unyah upas urang )
-
-  IRREGULARS_FOR_WORDS_BEGINS_WITH_P  = %w(
-    adam ahat akai amer anas ancang anggang anggil anjat antul asang asti atuhi
-    ecah ecat elihara eluk ercik eriksa erintah esan ikir ilah ilih injam inta
-    isah otong otret uja uji ukul usat utar-balik utus )
-
-  IRREGULARS_FOR_WORDS_BEGINS_WITH_N = %w( aas ada adi afi afsu aif aik akal akoda
-    alar ama anti arasi asab asib asional atif asehat asihat atural etral ikah )
-
-  IRREGULARS_FOR_WORDS_BEGINS_WITH_R = %w( aba abak aban abas abat abet abit
-    abuk abun abung abut acak acau acik acuh acun adah adai adak adang adiasi
-    adikal adio adu aga agam agas agi agu aguk ahap ahasia ahat ahim ahmat aih
-    aja ajah ajalela ajam ajang ajin ajuk ajut akap akat akit aksi akuk akus
-    akut akyat alat alip amah amahtamah amah-tamah amai amal ambah ambai ambak
-    amban ambang ambat ambeh ambu ambut amin ampai ampak ampang ampas ampat
-    amping ampok ampung ampus amu amus anap anca ancah ancak ancang ancap
-    ancu ancung anda andai andak andat andau andek anduk andung angah angai
-    angak anggah asa usak )
-
-  IRREGULAR_PREFIX_CHARACTERS_ON_WORDS = {
-    'meng' => IRREGULARS_FOR_WORDS_BEGINS_WITH_K,
-    'peng' => IRREGULARS_FOR_WORDS_BEGINS_WITH_K,
-    'mem' => IRREGULARS_FOR_WORDS_BEGINS_WITH_P,
-    'pem' => IRREGULARS_FOR_WORDS_BEGINS_WITH_P,  }
-
-  IRREGULAR_WORDS_ENDS_WITH_COMMON_CHARACTERS = {
-    'kah' => %w(  bengkah berkah bingkah bongkah cekah firkah halakah halkah
-                  harakah ingkah jangkah jerkah kalah kekah kelakah kerakah kerkah
-                  khalikah langkah lukah markah mukah musyarakah nafkah naskah
-                  nikah pangkah rakah rekah rengkah sedekah sekah serakah serkah
-                  sungkah takah tekah telingkah tingkah tongkah ),
-
-    'lah' => %w(  balah belah beslah bilah celah galah islah istilah jumlah
-                  kalah kelah kilah lalah lelah makalah malah masalah
-                  muamalah mujadalah mukabalah olah onslah oplah pecahbelah
-                  pecah-belah pilah milah sekolah rihlah risalah salah serlah
-                  silsilah sudah sulalah telah tulah ulah uzlah walah wasilah ),
-
-    'pun' => %w(  ampun depun himpun lapun rapun rumpun ),
-
-    'ku'  => %w(  awabeku baku bangku beku beluku biku buku ceku ciku cuku deku
-                  jibaku kaku laku leku liku luku paku pangku peku perilaku saku
-                  siku suku teleku terungku tungku waluku ),
-
-    'mu'  => %w(  ilmu jamu jemu kemu ramu selumu tamu temu ),
-
-    'nya' => %w(  tanya  ),
-  }
 
   REMOVED_KE    = 1
   REMOVED_PENG  = 2
@@ -143,6 +93,8 @@ module IndonesianStemmer
       end
 
       def remove_suffix(word)
+        return word if ambiguous_with_suffices_ending_words?(word)
+
         @number_of_syllables ||= total_syllables(word)
 
         SUFFIX_CHARACTERS.each do |character|
@@ -192,13 +144,12 @@ module IndonesianStemmer
 
         def remove_characters_matching_collection(word, collection, position)
           collection.each do |characters|
-            if send("#{position}s_with?", word, word.size, characters)
-              unless ambiguous_with_characters?(word, characters, position)
-                @flags ||= collection_for(characters, 'removed')
-                reduce_syllable
-                slice_word_at_position(word, characters.size, position)
-                return word
-              end
+            if match_position_and_not_ambiguous_with_characters?(word, characters, position)
+              next if characters == 'mem' && is_vowel?(word[characters.size])
+              @flags ||= collection_for(characters, 'removed')
+              reduce_syllable
+              slice_word_at_position(word, characters.size, position)
+              return word
             end
           end
 
@@ -211,32 +162,24 @@ module IndonesianStemmer
         end
 
         def remove_and_substitute_characters_matching_collection(word, collection, position)
-          word_size = word.size
           collection.each do |characters|
-            characters_size = characters.size
-            if send("#{position}s_with?", word, word_size, characters) &&
-                  word_size > characters_size && is_vowel?(word[characters_size])
-
-              if WITH_VOWEL_SUBSTITUTION_PREFIX_CHARACTERS.include?(characters) ||
-                    contains_irregular_prefix?(word, characters)
-
-                @flags ||= collection_for(characters, 'removed')
-                reduce_syllable
-                word = substitute_word_character(word, characters)
-                slice_word_at_position( word,
-                                        characters_size-1,
-                                        :start )
-                return word
-              end
+            if matching_characters_requires_substitution?(word, characters, position)
+              @flags ||= collection_for(characters, 'removed')
+              reduce_syllable
+              word = substitute_word_character(word, characters)
+              slice_word_at_position( word,
+                                      characters.size-1,
+                                      :start )
+              return word
             end
           end
         end
 
         def contains_irregular_prefix?(word, characters)
-          if IRREGULAR_PREFIX_CHARACTERS_ON_WORDS.keys.include?(characters)
+          if IrregularWords::ON_PREFIX_CHARACTERS.keys.include?(characters)
             chopped_word_match_words_collection?(
               word[characters.size, word.size],
-              IRREGULAR_PREFIX_CHARACTERS_ON_WORDS[characters] )
+              IrregularWords::ON_PREFIX_CHARACTERS[characters] )
           end
         end
 
@@ -250,7 +193,7 @@ module IndonesianStemmer
             's'
           when %w(men pen).include?(characters)
             (chopped_word_match_words_collection?(
-                word[characters.size, word.size], IRREGULARS_FOR_WORDS_BEGINS_WITH_N
+                word[characters.size, word.size], IrregularWords::BEGINS_WITH_N
               )
             )? 'n' : 't'
           when %w(meng peng).include?(characters)
@@ -266,12 +209,12 @@ module IndonesianStemmer
           if position == :start
             if characters == 'per'
               chopped_word_match_words_collection?(word[3..-1],
-                  IRREGULARS_FOR_WORDS_BEGINS_WITH_R )
+                  IrregularWords::BEGINS_WITH_R )
             else
               return false
             end
           else
-            IRREGULAR_WORDS_ENDS_WITH_COMMON_CHARACTERS[characters].any? do |ambiguous_word|
+            IrregularWords::ENDS_WITH_COMMON_CHARACTERS[characters].any? do |ambiguous_word|
               # To differentiate 'mobilmu' with 'berilmu'
               return false unless %w(me be pe).include?(word[0,2])
               # The rest is ok
@@ -280,6 +223,33 @@ module IndonesianStemmer
           end
         end
 
+        def ambiguous_with_suffices_ending_words?(word)
+          IrregularWords::ENDS_WITH_SUFFIX_CHARACTERS.include?(word)
+        end
+
+        def match_position_and_not_ambiguous_with_characters?(word, characters, position)
+          send("#{position}s_with?", word, word.size, characters) &&
+              !ambiguous_with_characters?(word, characters, position)
+        end
+
+        def match_characters_position_followed_by_vowel?(word, characters, position)
+          word_size = word.size
+          characters_size = characters.size
+
+          send("#{position}s_with?", word, word_size, characters) &&
+              word_size > characters_size && is_vowel?(word[characters_size])
+        end
+
+        def substitution_required?(word, characters)
+          WITH_VOWEL_SUBSTITUTION_PREFIX_CHARACTERS.include?(characters) ||
+              contains_irregular_prefix?(word, characters)
+        end
+
+        def matching_characters_requires_substitution?(word, characters, position)
+          match_characters_position_followed_by_vowel?(word, characters, position) &&
+              substitution_required?(word, characters)
+        end
+
         def reduce_syllable
           @number_of_syllables -= 1
         end

data/lib/indonesian_stemmer/version.rb

@@ -1,3 +1,3 @@
 module IndonesianStemmer
-  VERSION = "0.1.1"
+  VERSION = "0.2.0"
 end

data/spec/indonesian_stemmer_spec.rb

@@ -68,6 +68,8 @@ describe IndonesianStemmer do
     describe "'me'" do
       it { should_stem 'merusak', 'rusak'}
       it { should_stem 'melayang', 'layang'}
+      it { should_stem 'memasak', 'masak'}
+      it { should_stem 'memandikan', 'mandi'}
     end
 
     describe "'peng'" do

data/spec/lib/indonesian_stemmer/morphological_utility_spec.rb

@@ -263,6 +263,10 @@ describe IndonesianStemmer::MorphologicalUtility do
             should_transform(:remove_first_order_prefix, 'membangun', 'bangun')
           end
 
+          it "'mem' followed by vowel" do
+            should_transform(:remove_first_order_prefix, 'memilih', 'pilih')
+          end
+
           it "'me'" do
             should_transform(:remove_first_order_prefix, 'melukis', 'lukis')
           end
@@ -630,5 +634,15 @@ describe IndonesianStemmer::MorphologicalUtility do
         end
       end
     end
+
+    describe 'should not remove suffix characters for words ending with them' do
+      it "'kan'" do
+        should_not_transform(:remove_suffix, 'majikan')
+      end
+
+      it "'i'" do
+        should_not_transform(:remove_suffix, 'pandai')
+      end
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: indonesian_stemmer
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-04-07 00:00:00.000000000 Z
+date: 2013-04-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -61,6 +61,18 @@ files:
 - Rakefile
 - indonesian_stemmer.gemspec
 - lib/indonesian_stemmer.rb
+- lib/indonesian_stemmer/irregular_words.rb
+- lib/indonesian_stemmer/irregular_words/akhiran-i.txt
+- lib/indonesian_stemmer/irregular_words/k.txt
+- lib/indonesian_stemmer/irregular_words/kah.txt
+- lib/indonesian_stemmer/irregular_words/ku.txt
+- lib/indonesian_stemmer/irregular_words/lah.txt
+- lib/indonesian_stemmer/irregular_words/mu.txt
+- lib/indonesian_stemmer/irregular_words/n.txt
+- lib/indonesian_stemmer/irregular_words/nya.txt
+- lib/indonesian_stemmer/irregular_words/p.txt
+- lib/indonesian_stemmer/irregular_words/pun.txt
+- lib/indonesian_stemmer/irregular_words/r.txt
 - lib/indonesian_stemmer/morphological_utility.rb
 - lib/indonesian_stemmer/stemmer_utility.rb
 - lib/indonesian_stemmer/version.rb
@@ -82,7 +94,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 2800268474079069831
+      hash: 550012699463393318
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -91,7 +103,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 2800268474079069831
+      hash: 550012699463393318
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.25