llt-tokenizer 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3e46c64430c5caec2c91a4bb82b4322a9b02579a
-  data.tar.gz: 2f746527437f1bdbd683e0033a2597300f7635d1
+  metadata.gz: 17d75e05ecdf64cd93e8de239e57652b4a6375ba
+  data.tar.gz: b21ab0737044b952a9642a68a9aae48a4d615d4e
 SHA512:
-  metadata.gz: 7366d89f3b48de21d266368690cfb927dbb408df562f5b677d87c632f3c5081972d87fd0be2c2da74149936915e03c10873bd881a33fe4198ea766a1d3197279
-  data.tar.gz: cd55bd5af33d9cf077228ccb08559ad2fea87329d2982bf32fcdb31f11cc851255e9b26ed315058858ad44f722c535378d26018a0dec1f5d0e2982c1d96b47c5
+  metadata.gz: afadf19b6eb1a7f45dca79631efa57ceb7359d286523d2979658208decffea5d094fdcd4fcc9b3e02f788390bd3e2eeb28c415f3ad7ea97efcdbdf28602c7d2d
+  data.tar.gz: f752b02680802aafb6305490b139dc0b9ca2e3e35e9b48ca2610f2039e57959571ac432e2b7fa3b4b9e1f05abdc54c9ef8e99f42a2927a2ee25294ecad908fd9

data/Gemfile

@@ -9,6 +9,7 @@ gem 'llt-core', git: 'git@github.com:latin-language-toolkit/llt-core.git'
 gem 'llt-core_extensions', git: 'git@github.com:latin-language-toolkit/llt-core_extensions.git'
 gem 'llt-constants', git: 'git@github.com:latin-language-toolkit/llt-constants.git'
 gem 'llt-db_handler', git: 'git@github.com:latin-language-toolkit/llt-db_handler.git'
+gem 'llt-db_handler-stub', git: 'git@github.com:latin-language-toolkit/llt-db_handler-stub.git'
 gem 'llt-helpers', git: 'git@github.com:latin-language-toolkit/llt-helpers.git'
 
 # Dependencies of db_handler

data/lib/llt/tokenizer.rb

@@ -140,8 +140,8 @@ module LLT
 
   ######################
 
-    WORDS_ENDING_WITH_QUE = /^([qc]u[ei].*que|qu[ao]que|itaque|atque|ut[er].*que|utcumque|plerumque|denique|undique)$/i # neque taken out!
-    WORDS_ENDING_WITH_NE  = /^(omne|sine|bene|paene)$/i
+    WORDS_ENDING_WITH_QUE = /^((un.{1,3})?[qc]u[aei].*que|qu[ao]que|itaque|atque|ut[er].*que|.*cumque|pler(.{1,2}|[oa]rum)que|denique|undique|usque)$/i # neque taken out!
+    WORDS_ENDING_WITH_NE  = /^(omne|sine|bene|paene|iuvene)$/i
     WORDS_ENDING_WITH_VE  = /^(sive|neve)$/i
 
     # laetusque  to -que laetus
@@ -151,6 +151,10 @@ module LLT
     # but
     #
     # uterque, institutione, sive et al. remain
+    #
+    # iuvene might come as a suprise in these lists - it's a hack, but
+    # special because it has ve and ne - both would get split. Such words
+    # might be so rare that we postpone proper handling for now
 
     ENCLITICS = %w{ que ne ve c }
     def split_enklitika_and_change_their_position
@@ -205,8 +209,8 @@ module LLT
       # # TODO 27.11.13 14:15 by LFDM
       # Implement caching here
       ne_corrections
-      que_corrections
       ve_corrections
+      que_corrections
     end
 
     def que_corrections
@@ -243,8 +247,7 @@ module LLT
           entries = []
           entries += lookup(orig_el, :noun, :nom)           if orig_el =~ /io$/   # actio-ne ratio-ne
           entries += lookup(orig_el + "n", :persona, :stem) if orig_el =~ /o$/    # Plato-ne Cicero-ne Solo-ne
-          entries += lookup(orig_el + "n", :noun, :stem)    if orig_el =~ /d?i$/  # fortitudi-ne ratio-ne libidi-ne homi-ne
-          entries += lookup(orig_el + "n", :noun, :stem)    if orig_el =~ /mi$/   # flumi-ne agmi-ne
+          entries += lookup(orig_el + "n", :noun, :stem, [3, 33])    if orig_el =~ /[ei]$/ # fortitudi-ne ratio-ne libidi-ne homi-ne fi-ne agmi-ne iuve-ne
           entries += lookup(orig_el + "n", :noun, :stem, 2)                       # domi-ne
           entries += lookup(orig_el + "n", :adjective, :stem, [1,3])              # communis commune, or bonus
 
@@ -272,11 +275,11 @@ module LLT
           entries = []
           entries += lookup(orig_el + 'v',  :adjective, :stem, 1)
           entries += lookup(orig_el + 'v',  :adjective, :stem, 3)
-          entries += lookup(orig_el + 'v',  :noun,      :stem, [2, 5])
+          entries += lookup(orig_el + 'v',  :noun,      :stem, [2, 33, 5])
+          entries += lookup(orig_el + 'v',  :persona,   :stem, 3)
           entries += lookup(orig_el + 've', :verb,      :pr,   2)
           entries += lookup(orig_el + 'v',  :verb,      :pr,   [3, 5]) # not sure if such a word of 5 exists
 
-
           if entries.any?
             corrections << i - corrections.size
           end
@@ -346,7 +349,6 @@ module LLT
     ABBR_NAME_WITH_DOT       = /^(#{NAMES_PIPED})\.$/
     ROMAN_DATE_EXPR_WITH_DOT = /^(#{DATES_PIPED})\.$/
     PUNCT_ITSELF             = Regexp.new("^(?:#{PUNCTUATION.source})$")
-    XML_TAG                  = /<\/?.+?>/
 
     def create_tokens
       # call #to_a is to retrieve (and align) optional metrical data

data/lib/llt/tokenizer/version.rb

@@ -1,5 +1,5 @@
 module LLT
   class Tokenizer
-    VERSION = "0.0.4"
+    VERSION = "0.0.5"
   end
 end

data/spec/lib/llt/tokenizer_spec.rb

@@ -165,13 +165,23 @@ describe LLT::Tokenizer do
           "denique"   => "denique",
           "itaque"    => "itaque",
           "plerumque" => "plerumque",
+          "plerosque" => "plerosque",
+          "plerique"  => "plerique",
+          "plerarumque"  => "plerarumque",
           "quaque"    => "quaque",
           "quemque"   => "quemque",
           "undique"   => "undique",
           "uterque"   => "uterque",
           "utriusque" => "utriusque",
           "utcumque"  => "utcumque",
+          "usque"     => "usque",
           "bonus laetusque et latus altusque" => "bonus -que laetus et latus -que altus",
+          "quantumcumque" => "quantumcumque",
+          "quantulacumque" => "quantulacumque",
+          "unusquisque" => "unusquisque",
+          "quisque" => "quisque",
+          "quaeque" => "quaeque",
+          "uniuscuiusque" => "uniuscuiusque"
         }
 
         examples.each do |example, expected|
@@ -189,11 +199,15 @@ describe LLT::Tokenizer do
           # frequent patterns in third declension nouns
           "ratione"      => "ratione",
           "magnitudine"  => "magnitudine",
+          "iactatione"   => "iactatione",
           "Platone"      => "Platone",
           "libidine"     => "libidine",
           "Solone"       => "Solone",
           "homine"       => "homine",
           "flumine"      => "flumine",
+          "fine"         => "fine",
+          "iuvene"       => "iuvene",
+          "sanguine"     => "sanguine",
 
           # frequent patterns in third declension adjective
           "commune"    => "commune",
@@ -232,7 +246,9 @@ describe LLT::Tokenizer do
           'suave'    => 'suave',
           'vive'     => 'vive',
           'move'     => 'move',
-          'neve'     => 'neve'
+          'neve'     => 'neve',
+          'cive'     => 'cive',
+          'Iove'     => 'Iove',
         }
 
         examples.each do |example, expected|

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: llt-tokenizer
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - LFDM
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-08 00:00:00.000000000 Z
+date: 2014-02-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler