llt-tokenizer 0.0.7 → 0.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cd490c0611dc9cd5ed2f2aad95631b500e7e7d35
-  data.tar.gz: fe8162ded4cecfda383f3919576ec19ca14f5a38
+  metadata.gz: 6ea54a34c5f463b52600e854cd9e9084a5599af7
+  data.tar.gz: 13cfe38eb69014a5d855e4e0d44a9d54991bc855
 SHA512:
-  metadata.gz: a8a90a133ccd0c27fbbd20e64df0a8e560ad183010ae55716362e0909c737a6d5359c51a647da387ddade71a3bd50545381c899cd3fcea095001ef0e27c42483
-  data.tar.gz: 2d81412a66b473206d23aec3128d3321ff736f3a9d5ab8fca0349bcc347e8d692b3ee33d0fd2c67f87f031335113c1a54caef80b5aa0a4eaca4c5bd2320d6778
+  metadata.gz: 7fcdbcfc60c63cbec28a0aa37044cddb22280ec4f4862c39bf3abc658619f273185589714c5d4ac27eb4c7a881c350047a499dba68c43e932e0cd18b02b8670d
+  data.tar.gz: 1c5bf2c479b0ff8e49358a1d53e5bb597ab5157d6981d65541eec6189e95d8fb57467bc62fd406d86ed1b4b473139238438cc589a4b2b150ec51798f2a837942

data/lib/llt/tokenizer.rb

@@ -152,7 +152,7 @@ module LLT
 
     WORDS_ENDING_WITH_QUE = /^((un.{1,3})?[qc]u[aei].*que|qu[ao]que|itaque|atque|ut[er].*que|.*cumque|pler(.{1,2}|[oa]rum)que|denique|undique|usque)$/i # neque taken out!
     WORDS_ENDING_WITH_NE  = /^(omne|sine|bene|paene|iuvene|siccine)$/i # generalize these words and start to look for them in the db, especiialy for adverbs
-    WORDS_ENDING_WITH_VE  = /^(sive|neve)$/i
+    WORDS_ENDING_WITH_VE  = /^()$/i # formerly had neve and sive, which we split now
 
     # laetusque  to -que laetus
     # in eoque   to -que in eo
@@ -169,7 +169,7 @@ module LLT
     ENCLITICS = %w{ que ne ve c }
     def split_enklitika_and_change_their_position
       split_with_force
-      split_nec_and_oute
+      split_frequent_enclitics # like latin c, ve or greek te, de
       make_frequent_corrections
     end
 
@@ -202,22 +202,24 @@ module LLT
       "#{@enclitics_marker}#{val}"
     end
 
-    def split_nec_and_oute
-      nec_indices  = []
-      oute_indices = []
+    ENCLITICS_MAP = {
+      /^(nec)$/i => 'c',
+      /^(ne|se)u$/i => 'u',
+      /^(nisi)$/i => 'si',
+      /^(οὐ|μή|εἰ)τε$/i => 'τε',
+      /^(οὐ|μή)δε$/i => 'δε',
+    }
+    def split_frequent_enclitics
+      container = []
       @worker.each_with_index do |token, i|
-        case token
-        when /^nec$/i
-          token.slice!(-1)
-          nec_indices << (i + nec_indices.size + @shift_range)
-        when /^οὐτε$/i
-          token.slice!(-2, 2)
-          oute_indices << (i + oute_indices.size + @shift_range)
+        ENCLITICS_MAP.each do |regex, encl|
+          if token.match(regex)
+            token.slice!(-encl.length, encl.length)
+            container << [encl, (i + container.size + @shift_range)]
+          end
         end
       end
-
-      nec_indices.each  { |i| @worker.insert(i, enclitic('c')) }
-      oute_indices.each { |i| @worker.insert(i, enclitic('τε')) }
+      container.each { |encl, i|@worker.insert(i, enclitic(encl)) }
     end
 
     def make_frequent_corrections

data/lib/llt/tokenizer/version.rb

@@ -1,5 +1,5 @@
 module LLT
   class Tokenizer
-    VERSION = "0.0.7"
+    VERSION = "0.0.8"
   end
 end

data/spec/lib/llt/tokenizer_spec.rb

@@ -242,14 +242,14 @@ describe LLT::Tokenizer do
 
       context "when confronted with -ve" do
         examples = {
-          'sive'     => 'sive',
+          'sive'     => '-ve si',
+          'neve'     => '-ve ne',
           'pluresve' => '-ve plures',
           'aestive'  => 'aestive',
           'serve'    => 'serve',
           'suave'    => 'suave',
           'vive'     => 'vive',
           'move'     => 'move',
-          'neve'     => 'neve',
           'cive'     => 'cive',
           'Iove'     => 'Iove',
         }
@@ -261,9 +261,49 @@ describe LLT::Tokenizer do
         end
       end
 
+      context "when confronted with -u" do
+        examples = {
+          'seu' => '-u se',
+          'neu' => '-u ne'
+        }
+
+        examples.each do |example, expected|
+          it "transforms #{example} to #{expected}" do
+            enklitika_test(example).should be_transformed_to expected
+          end
+        end
+      end
+
+      context "when confronted with -si" do
+        examples = {
+          'nisi' => '-si ni'
+        }
+
+        examples.each do |example, expected|
+          it "transforms #{example} to #{expected}" do
+            enklitika_test(example).should be_transformed_to expected
+          end
+        end
+      end
+
       context "when confronted with -τε" do
         examples = {
-          'οὐτε'  => '-τε οὐ'
+          'οὐτε'  => '-τε οὐ',
+          'μήτε'  => '-τε μή',
+          'εἰτε'  => '-τε εἰ'
+        }
+
+        examples.each do |example, expected|
+          it "transforms #{example} to #{expected}" do
+            enklitika_test(example).should be_transformed_to expected
+          end
+        end
+      end
+
+      context "when confronted with -δε" do
+        examples = {
+          'οὐδε'  => '-δε οὐ',
+          'μήδε'  => '-δε μή'
         }
 
         examples.each do |example, expected|

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: llt-tokenizer
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.8
 platform: ruby
 authors:
 - LFDM
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-17 00:00:00.000000000 Z
+date: 2014-08-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler