llt-segmenter 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 201cbc98ca192041429641ef0e3c0fcf128a3654
-  data.tar.gz: 1f61cb110d27d4b60427f16b1c7eb58576c67eb0
+  metadata.gz: f6c90686915f9e02706f88b650695a4b40aa6867
+  data.tar.gz: 67ed2ade8bb50c0419a82f18b40eeef031c7a243
 SHA512:
-  metadata.gz: 76017f1fc143d0d6d190b341218c3272f71ad230a1c81496ab9813a94a285262f3c15e73ef73570fc8a5b5d07ceb726b7e2ad9e046f9c54a8d61576017c0aac8
-  data.tar.gz: 0915e76fac1f68d23b9454c010fc5820b021457ff4b53cf8bb0bf98a05b73cffc21b888a0169d14d331a571812c5027e26e7dd56a07dc6936be7538160dff099
+  metadata.gz: e38adead709637f4520233ead966d000c332b8b356f4e2d827ae40b3ce31d6e2a0e6969762b57393bcb02c9a9b647e1e617a5bd81ab20712663ac525de7a5062
+  data.tar.gz: 50fedc9e52883f8458a437a6f6ff6252bbe6e1f6d78cdfb59d5b78873a27da3dfc40481a1008cf7278493196041cc42a5beaa40be2faef9071e5f36aad946769

data/Gemfile

@@ -6,9 +6,9 @@ gem 'pry'
 
 gem 'coveralls', require: false
 
-gem 'llt-core', git: 'https://github.com/latin-language-toolkit/llt-core.git'
-gem 'llt-constants', git: 'https://github.com/latin-language-toolkit/llt-constants.git'
-gem 'llt-logger', git: 'https://github.com/latin-language-toolkit/llt-logger.git'
+gem 'llt-core', git: 'git://github.com/latin-language-toolkit/llt-core.git'
+gem 'llt-constants', git: 'git://github.com/latin-language-toolkit/llt-constants.git'
+gem 'llt-logger', git: 'git://github.com/latin-language-toolkit/llt-logger.git'
 
 platform :jruby do
   gem 'jruby-httpclient'

data/lib/llt/segmenter/api.rb

@@ -5,6 +5,7 @@ require 'llt/core/api'
 
 class Api < Sinatra::Base
   register Sinatra::RespondWith
+  register LLT::Core::Api::VersionRoutes
   helpers LLT::Core::Api::Helpers
 
   get '/segment' do
@@ -17,4 +18,6 @@ class Api < Sinatra::Base
       f.xml { to_xml(sentences, params) }
     end
   end
+
+  add_version_route_for('/segment', dependencies: %i{ Core Segmenter })
 end

data/lib/llt/segmenter/version.rb

@@ -1,5 +1,5 @@
 module LLT
   class Segmenter
-    VERSION = "0.0.4"
+    VERSION = "0.0.5"
   end
 end

data/lib/llt/segmenter/version_info.rb

@@ -0,0 +1,7 @@
+module LLT
+  class Segmenter
+    class VersionInfo
+      include Core::Versioner
+    end
+  end
+end

data/lib/llt/segmenter.rb

@@ -2,6 +2,8 @@ require "llt/constants"
 require "llt/core"
 require "llt/logger"
 require "llt/sentence"
+require "llt/segmenter/version"
+require "llt/segmenter/version_info"
 
 module LLT
   class Segmenter
@@ -30,7 +32,7 @@ module LLT
     AWB = ALL_ABBRS_PIPED.split('|').map { |abbr| "(?<=\\s|^|>)#{abbr}" }.join('|')
     # the xml escaped characters cannot be refactored to something along
     # &(?:amp|quot); - it's an invalid pattern in the look-behind
-    SENTENCE_CLOSER = /(?<!#{AWB})\.(?!\.)|[\?!:]|((?<!&amp|&quot|&apos|&lt|&gt);)/
+    SENTENCE_CLOSER = /(?<!#{AWB})\.(?!\.)|[\?!:·]|((?<!&amp|&quot|&apos|&lt|&gt);)/
     DIRECT_SPEECH_DELIMITER = /['"”]|&(?:apos|quot);/
     TRAILERS = /\)|\s*<\/.*?>/
 
@@ -51,8 +53,14 @@ module LLT
       @indexing = parse_option(:indexing, options)
       @id = 0 if @indexing
 
+      # newline_boundary is only active when we aren't working with xml!
       nl_boundary  = parse_option(:newline_boundary, options)
-      @sentence_closer = Regexp.union(SENTENCE_CLOSER, /\n{#{nl_boundary}}/)
+
+      @sentence_closer = build_sentence_closer_regexp(nl_boundary)
+    end
+
+    def build_sentence_closer_regexp(nl_boundary)
+      @xml ? SENTENCE_CLOSER : Regexp.union(SENTENCE_CLOSER, /\n{#{nl_boundary}}/)
     end
 
     # Used to normalized wonky whitespace in front of or behind direct speech
@@ -115,9 +123,16 @@ module LLT
 
     def scan_through_string(scanner, sentences = [])
       while scanner.rest?
+        loop_guard = scanner.pos
+
         sentence = scan_until_next_sentence(scanner, sentences)
 
-        rebuild_xml_tags(scanner, sentence, sentences) if @xml
+        raise if scanner.pos == loop_guard
+
+        if @xml
+          rebuild_xml_tags(scanner, sentence, sentences)
+          take_all_closing_tags(scanner, sentence)
+        end
         sentence << trailing_delimiters(scanner)
 
         sentence.strip!
@@ -130,6 +145,10 @@ module LLT
       sentences
     end
 
+    def scan_to_first_real_text(scanner)
+      scanner.scan_until(/<.*?>\s*(?=\w)/)
+    end
+
     def scan_until_next_sentence(scanner, sentences)
       scanner.scan_until(@sentence_closer) ||
         rescue_no_delimiters(sentences, scanner)
@@ -160,6 +179,18 @@ module LLT
       ! sentence.match(/#{@sentence_closer}\s*<.*?>$/)
     end
 
+    def take_all_closing_tags(scanner, sentence)
+      if closing_tags_only?(scanner.rest)
+        sentence << scanner.rest
+        scanner.terminate
+      end
+    end
+
+    def closing_tags_only?(str)
+      str.match(/\A(\s*<\/.*?>\s*|\s*<.*?\/>\s*)+\z/)
+    end
+
+
     def rescue_no_delimiters(sentences, scanner)
       if sentences.any?
         # broken off texts

data/llt-segmenter.gemspec

@@ -20,7 +20,7 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rake"
-  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "rspec", "~> 2.14"
   spec.add_development_dependency "simplecov", "~> 0.7"
   spec.add_dependency "llt-core"
   spec.add_dependency "llt-constants"

data/spec/fixtures/petrov_eleg01_cleaned.xml

@@ -0,0 +1,203 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-model href="http://www.stoa.org/epidoc/schema/latest/tei-epidoc.rng" type="application/xml" schematypens="http://relaxng.org/ns/structure/1.0"?>
+<TEI xmlns="http://www.tei-c.org/ns/1.0">
+   <text xml:lang="lat">
+      <body>
+         <div type="edition" subtype="poesis-elegia" n="urn:cts:croala:petrov02.eleg01.lat1">
+            <lg met="elegiacum">
+               <l n="1">Conjugis ut carae patrio mens debita caelo</l>
+
+               <l n="2"> Pars melior, miseram laeta reliquit humum:</l>
+
+               <l n="3">Postquam, illa rapta, simul omnis rapta voluptas</l>
+
+               <l n="4"> Est mihi (jam ex illo tempore mensis abit)</l>
+
+               <l n="5">Non nisi perpetuos fundunt mea lumina fletus;</l>
+
+               <l n="6">Lux sequitur noctem tristis, et umbra diem.</l>
+
+               <l n="7">Pabula sunt <sic>lacrymae</sic>, <sic>lacrymae</sic> sunt pocula, sed quae</l>
+
+               <l n="8"> Plus, Medea, tuo gramine fellis habent.</l>
+
+               <l n="9">Sic me qui pascit, simul enecat humor, et hospes</l>
+
+               <l n="10"> Ipse mei cordis, cor mihi luctus edit.</l>
+
+               <l n="11">Ac veluti turtur, sociam cui barbarus auceps</l>
+
+               <l n="12"> Exceptam structis perdidit insidiis,</l>
+
+               <l n="13">Si quam forte videt sine vite, et frondibus ulmum,</l>
+
+               <l n="14"> Aequalem sorti, consimilemque suae,</l>
+
+               <l n="15">Flectit iter, ramoque sedens miserabilis ales</l>
+
+               <l n="16"> Gutture subrauco nil nisi triste gemit.</l>
+
+               <l n="17">Non illum exhilarat facies pulcherrima Veris,</l>
+
+               <l n="18"> Nulla sibi in notis pabula quaerit agris:</l>
+
+               <l n="19">Non sociae possunt volucres abducere ramo,</l>
+
+               <l n="20"> Ad prope labentes non sitis urget aquas.</l>
+
+               <l n="21">Sic ego. Sic vitam sine te, dulcissima conjux,</l>
+
+               <l n="22"> Si vitae haec nomen vita meretur, ago.</l>
+
+               <l n="23">Sola queri misero, sola est mihi flere voluptas,</l>
+
+               <l n="24"> Sola loci facies maesta, silensque placet.</l>
+
+               <l n="25">Non aures cantus, non fila loquacia mulcent;</l>
+
+               <l n="26"> Non oculos formae gratia, flosque rapit.</l>
+
+               <l n="27">Unam te in sylvis, unam in florentibus hortis,</l>
+
+               <l n="28">Per juga, per valles quaero, nec invenio.</l>
+
+               <l n="29">Nec magis Eurydice est Vati quaesita marito,</l>
+
+               <l n="30"> Tartareum quamvis viderit ille canem:</l>
+
+               <l n="31">Nec magis est Cephalo Procris defleta, videnti</l>
+
+               <l n="32"> Deceptae errorem, flagitiumque manus;</l>
+
+               <l n="33">Quam totas ego te noctes, mea vita, diesque</l>
+
+               <l n="34"> Quaero, nec inventam maestus abesse queror.</l>
+
+               <l n="35">Et tamen ante oculos errat tua semper imago,</l>
+
+               <l n="36"> (Quid non fingit amans?) et tua verba sonant.</l>
+
+               <l n="37">Si qua avis in densis, Siren innoxia, lucis</l>
+
+               <l n="38"> Est audita mihi fundere dulce melos,</l>
+
+               <l n="39">Sisto gradum, et similis deceptus imagine vocis</l>
+
+               <l n="40"> Est, inquam, est cantus conjugis ille meae.</l>
+
+               <l n="41">Si quando ad fontes, aut ad vernantia prata,</l>
+
+               <l n="42"> Aut maris ad placidas me tulit error aquas,</l>
+
+               <l n="43">Hic locus est, dico, quem visere saepe solebat.</l>
+
+               <l n="44"> Quae mora (jam sol est ortus) abesse facit?</l>
+
+               <l n="45">Sed jam jam veniet; latet illa forte sub umbra,</l>
+
+               <l n="46"> Aut illi pietas est sua causa morae.</l>
+
+               <l n="47">Causa morae est certe pietas: nisi fallimur, haec est,</l>
+
+               <l n="48"> Fundentem ad superos quae videt hora preces.</l>
+
+               <l n="49">Mox sat ut illusum me liquit amabilis error,</l>
+
+               <l n="50"> Protinus ex oculis bina fluenta cadunt.</l>
+
+               <l n="51">Bina fluenta cadunt, quorum hinc dolor elicit unum,</l>
+
+               <l n="52"> Inde aliud, tanti causa doloris, amor.</l>
+
+               <l n="53">Meque ipsum incuso, quod sim tam stultus, et amens,</l>
+
+               <l n="54"> Et pascam aerumnas crudelitate meas:</l>
+
+               <l n="55">Rursus in errores tamen hos delabor, et hujus</l>
+
+               <l n="56"> Erroris rursus paenitet esse reum.</l>
+
+               <l n="57">Sic pugnant mea vota meis contraria votis,</l>
+
+               <l n="58"> Nec placet, heu! misero quod modo dulce fuit.</l>
+
+               <l n="59">Nec quod sim discors, angit modo; saevius angit</l>
+
+               <l n="60"> Vivere me longos te sine posse dies.</l>
+
+               <l n="61">Ah! ubi sunt voces illae, et mea fortia verba?</l>
+
+               <l n="62"> Ah! ubi, quae verbis debet inesse, fides?</l>
+
+               <l n="63">Me quoque rapturam subito, quae te hora tulisset,</l>
+
+               <l n="64"> Et pariter praedam mortis utrumque fore?</l>
+
+               <l n="65">Ecce tamen vivo, nec post nova cornua Phoebes</l>
+
+               <l n="66"> Vis me maeroris perdere longa potest.</l>
+
+               <l n="67">Heu! quae dura silex, quod inexsuperabile robur,</l>
+
+               <l n="68"> Quod ferrum, et triplex aes mihi pectus obit?</l>
+
+               <l n="69">Vivo equidem, vivo, sed morte est tristior ipsa,</l>
+
+               <l n="70"> Quae sine te, conjux, vita relicta mihi est.</l>
+
+               <l n="71">At tu nunc choreis Natorum immixta tuorum,</l>
+
+               <l n="72"> Qui (prona) facili ad Superos te praeiere (via) gradu,</l>
+
+               <l n="73">Plena Deo frueris, nec, quae tibi parta, bonorum</l>
+
+               <l n="74"> Amittendorum te timor ullus habet.</l>
+
+               <l n="75">Nam tua non tristes pietas te duxit in oras:</l>
+
+               <l n="76"> Debetur sedes non nisi laeta piis.</l>
+
+               <l n="77">Te plaga (credo equidem) summi plaga lucida caeli,</l>
+
+               <l n="78"> Te laeta aeterno vere vireta tenent.</l>
+
+               <l n="79">Ipsum ipsum Auctorem rerum, quem qui videt, ultra</l>
+
+               <l n="80"> Nil habet optandum, jam sine nube vides.</l>
+
+               <l n="81">Usque et ubique vides, at non saturata videndo</l>
+
+               <l n="82"> Illo oculos pascis; pressa sed usque fame es.</l>
+
+               <l n="83">Te vis implet opum, sed non (licet impleat) explet;</l>
+
+               <l n="84"> Excipit unum aliud, subsequiturque bonum.</l>
+
+               <l n="85">Non te humiles curae, non te mortalia tangunt;</l>
+
+               <l n="86"> Prae caelo, et stellis quam tibi sordet humus!</l>
+
+               <l n="87">Sordet humus certe. non sic tamen, ut tua nunquam</l>
+
+               <l n="88"> Ad miserum flectas lumina blanda virum;</l>
+
+               <l n="89">Audire aut flentem fugias, et saucia flentis,</l>
+
+               <l n="90"> Qua licet, admota corda fovere manu;</l>
+
+               <l n="91">Iactatumque diu ventisque undisque vocare</l>
+
+               <l n="92"> Ad laeta Eridani littora stelliferi.</l>
+
+               <l n="93">Quam tua sors felix, quam nostra simillima morti est,</l>
+
+               <l n="94"> Felle ego, tu Divum vesceris ambrosia.</l>
+
+               <l n="95">Non tamen invideo tua gaudia, sed miser opto,</l>
+
+               <l n="96"> Laetitiae consors quam prius esse tuae.</l>
+            </lg>
+         </div>
+      </body>
+   </text>
+</TEI>

data/spec/fixtures/petrov_eleg01_with_endless_loop.xml

@@ -0,0 +1,204 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-model href="http://www.stoa.org/epidoc/schema/latest/tei-epidoc.rng" type="application/xml" schematypens="http://relaxng.org/ns/structure/1.0"?>
+<TEI xmlns="http://www.tei-c.org/ns/1.0">
+   <text xml:lang="lat">
+      <body>
+         <div type="edition" subtype="poesis-elegia" n="urn:cts:croala:petrov02.eleg01.lat1">
+            <lg met="elegiacum">
+               <l n="1">Conjugis ut carae patrio mens debita caelo</l>
+
+               <l n="2"> Pars melior, miseram laeta reliquit humum:</l>
+
+               <l n="3">Postquam, illa rapta, simul omnis rapta voluptas</l>
+
+               <l n="4"> Est mihi (jam ex illo tempore mensis abit)</l>
+
+               <l n="5">Non nisi perpetuos fundunt mea lumina fletus;</l>
+
+               <l n="6">Lux sequitur noctem tristis, et umbra diem.</l>
+
+               <l n="7">Pabula sunt <sic>lacrymae</sic>, <sic>lacrymae</sic> sunt pocula, sed
+                  quae</l>
+
+               <l n="8"> Plus, Medea, tuo gramine fellis habent.</l>
+
+               <l n="9">Sic me qui pascit, simul enecat humor, et hospes</l>
+
+               <l n="10"> Ipse mei cordis, cor mihi luctus edit.</l>
+
+               <l n="11">Ac veluti turtur, sociam cui barbarus auceps</l>
+
+               <l n="12"> Exceptam structis perdidit insidiis,</l>
+
+               <l n="13">Si quam forte videt sine vite, et frondibus ulmum,</l>
+
+               <l n="14"> Aequalem sorti, consimilemque suae,</l>
+
+               <l n="15">Flectit iter, ramoque sedens miserabilis ales</l>
+
+               <l n="16"> Gutture subrauco nil nisi triste gemit.</l>
+
+               <l n="17">Non illum exhilarat facies pulcherrima Veris,</l>
+
+               <l n="18"> Nulla sibi in notis pabula quaerit agris:</l>
+
+               <l n="19">Non sociae possunt volucres abducere ramo,</l>
+
+               <l n="20"> Ad prope labentes non sitis urget aquas.</l>
+
+               <l n="21">Sic ego. Sic vitam sine te, dulcissima conjux,</l>
+
+               <l n="22"> Si vitae haec nomen vita meretur, ago.</l>
+
+               <l n="23">Sola queri misero, sola est mihi flere voluptas,</l>
+
+               <l n="24"> Sola loci facies maesta, silensque placet.</l>
+
+               <l n="25">Non aures cantus, non fila loquacia mulcent;</l>
+
+               <l n="26"> Non oculos formae gratia, flosque rapit.</l>
+
+               <l n="27">Unam te in sylvis, unam in florentibus hortis,</l>
+
+               <l n="28">Per juga, per valles quaero, nec invenio.</l>
+
+               <l n="29">Nec magis Eurydice est Vati quaesita marito,</l>
+
+               <l n="30"> Tartareum quamvis viderit ille canem:</l>
+
+               <l n="31">Nec magis est Cephalo Procris defleta, videnti</l>
+
+               <l n="32"> Deceptae errorem, flagitiumque manus;</l>
+
+               <l n="33">Quam totas ego te noctes, mea vita, diesque</l>
+
+               <l n="34"> Quaero, nec inventam maestus abesse queror.</l>
+
+               <l n="35">Et tamen ante oculos errat tua semper imago,</l>
+
+               <l n="36"> (Quid non fingit amans?) et tua verba sonant.</l>
+
+               <l n="37">Si qua avis in densis, Siren innoxia, lucis</l>
+
+               <l n="38"> Est audita mihi fundere dulce melos,</l>
+
+               <l n="39">Sisto gradum, et similis deceptus imagine vocis</l>
+
+               <l n="40"> Est, inquam, est cantus conjugis ille meae.</l>
+
+               <l n="41">Si quando ad fontes, aut ad vernantia prata,</l>
+
+               <l n="42"> Aut maris ad placidas me tulit error aquas,</l>
+
+               <l n="43">Hic locus est, dico, quem visere saepe solebat.</l>
+
+               <l n="44"> Quae mora (jam sol est ortus) abesse facit?</l>
+
+               <l n="45">Sed jam jam veniet; latet illa forte sub umbra,</l>
+
+               <l n="46"> Aut illi pietas est sua causa morae.</l>
+
+               <l n="47">Causa morae est certe pietas: nisi fallimur, haec est,</l>
+
+               <l n="48"> Fundentem ad superos quae videt hora preces.</l>
+
+               <l n="49">Mox sat ut illusum me liquit amabilis error,</l>
+
+               <l n="50"> Protinus ex oculis bina fluenta cadunt.</l>
+
+               <l n="51">Bina fluenta cadunt, quorum hinc dolor elicit unum,</l>
+
+               <l n="52"> Inde aliud, tanti causa doloris, amor.</l>
+
+               <l n="53">Meque ipsum incuso, quod sim tam stultus, et amens,</l>
+
+               <l n="54"> Et pascam aerumnas crudelitate meas:</l>
+
+               <l n="55">Rursus in errores tamen hos delabor, et hujus</l>
+
+               <l n="56"> Erroris rursus paenitet esse reum.</l>
+
+               <l n="57">Sic pugnant mea vota meis contraria votis,</l>
+
+               <l n="58"> Nec placet, heu! misero quod modo dulce fuit.</l>
+
+               <l n="59">Nec quod sim discors, angit modo; saevius angit</l>
+
+               <l n="60"> Vivere me longos te sine posse dies.</l>
+
+               <l n="61">Ah! ubi sunt voces illae, et mea fortia verba?</l>
+
+               <l n="62"> Ah! ubi, quae verbis debet inesse, fides?</l>
+
+               <l n="63">Me quoque rapturam subito, quae te hora tulisset,</l>
+
+               <l n="64"> Et pariter praedam mortis utrumque fore?</l>
+
+               <l n="65">Ecce tamen vivo, nec post nova cornua Phoebes</l>
+
+               <l n="66"> Vis me maeroris perdere longa potest.</l>
+
+               <l n="67">Heu! quae dura silex, quod inexsuperabile robur,</l>
+
+               <l n="68"> Quod ferrum, et triplex aes mihi pectus obit?</l>
+
+               <l n="69">Vivo equidem, vivo, sed morte est tristior ipsa,</l>
+
+               <l n="70"> Quae sine te, conjux, vita relicta mihi est.</l>
+
+               <l n="71">At tu nunc choreis Natorum immixta tuorum,</l>
+
+               <l n="72"> Qui (prona) facili ad Superos te praeiere (via) gradu,</l>
+
+               <l n="73">Plena Deo frueris, nec, quae tibi parta, bonorum</l>
+
+               <l n="74"> Amittendorum te timor ullus habet.</l>
+
+               <l n="75">Nam tua non tristes pietas te duxit in oras:</l>
+
+               <l n="76"> Debetur sedes non nisi laeta piis.</l>
+
+               <l n="77">Te plaga (credo equidem) summi plaga lucida caeli,</l>
+
+               <l n="78"> Te laeta aeterno vere vireta tenent.</l>
+
+               <l n="79">Ipsum ipsum Auctorem rerum, quem qui videt, ultra</l>
+
+               <l n="80"> Nil habet optandum, jam sine nube vides.</l>
+
+               <l n="81">Usque et ubique vides, at non saturata videndo</l>
+
+               <l n="82"> Illo oculos pascis; pressa sed usque fame es.</l>
+
+               <l n="83">Te vis implet opum, sed non (licet impleat) explet;</l>
+
+               <l n="84"> Excipit unum aliud, subsequiturque bonum.</l>
+
+               <l n="85">Non te humiles curae, non te mortalia tangunt;</l>
+
+               <l n="86"> Prae caelo, et stellis quam tibi sordet humus!</l>
+
+               <l n="87">Sordet humus certe. non sic tamen, ut tua nunquam</l>
+
+               <l n="88"> Ad miserum flectas lumina blanda virum;</l>
+
+               <l n="89">Audire aut flentem fugias, et saucia flentis,</l>
+
+               <l n="90"> Qua licet, admota corda fovere manu;</l>
+
+               <l n="91">Iactatumque diu ventisque undisque vocare</l>
+
+               <l n="92"> Ad laeta Eridani littora stelliferi.</l>
+
+               <l n="93">Quam tua sors felix, quam nostra simillima morti est,</l>
+
+               <l n="94"> Felle ego, tu Divum vesceris ambrosia.</l>
+
+               <l n="95">Non tamen invideo tua gaudia, sed miser opto,</l>
+
+               <l n="96"> Laetitiae consors quam prius esse tuae.</l>
+            </lg>
+         </div>
+      </body>
+   </text>
+</TEI>

data/spec/fixtures/petrov_eleg01_with_endless_loop_no_xml_header.xml

@@ -0,0 +1,202 @@
+<TEI xmlns="http://www.tei-c.org/ns/1.0">
+   <text xml:lang="lat">
+      <body>
+         <div type="edition" subtype="poesis-elegia" n="urn:cts:croala:petrov02.eleg01.lat1">
+            <lg met="elegiacum">
+               <l n="1">Conjugis ut carae patrio mens debita caelo</l>
+
+               <l n="2"> Pars melior, miseram laeta reliquit humum:</l>
+
+               <l n="3">Postquam, illa rapta, simul omnis rapta voluptas</l>
+
+               <l n="4"> Est mihi (jam ex illo tempore mensis abit)</l>
+
+               <l n="5">Non nisi perpetuos fundunt mea lumina fletus;</l>
+
+               <l n="6">Lux sequitur noctem tristis, et umbra diem.</l>
+
+               <l n="7">Pabula sunt <sic>lacrymae</sic>, <sic>lacrymae</sic> sunt pocula, sed
+                  quae</l>
+
+               <l n="8"> Plus, Medea, tuo gramine fellis habent.</l>
+
+               <l n="9">Sic me qui pascit, simul enecat humor, et hospes</l>
+
+               <l n="10"> Ipse mei cordis, cor mihi luctus edit.</l>
+
+               <l n="11">Ac veluti turtur, sociam cui barbarus auceps</l>
+
+               <l n="12"> Exceptam structis perdidit insidiis,</l>
+
+               <l n="13">Si quam forte videt sine vite, et frondibus ulmum,</l>
+
+               <l n="14"> Aequalem sorti, consimilemque suae,</l>
+
+               <l n="15">Flectit iter, ramoque sedens miserabilis ales</l>
+
+               <l n="16"> Gutture subrauco nil nisi triste gemit.</l>
+
+               <l n="17">Non illum exhilarat facies pulcherrima Veris,</l>
+
+               <l n="18"> Nulla sibi in notis pabula quaerit agris:</l>
+
+               <l n="19">Non sociae possunt volucres abducere ramo,</l>
+
+               <l n="20"> Ad prope labentes non sitis urget aquas.</l>
+
+               <l n="21">Sic ego. Sic vitam sine te, dulcissima conjux,</l>
+
+               <l n="22"> Si vitae haec nomen vita meretur, ago.</l>
+
+               <l n="23">Sola queri misero, sola est mihi flere voluptas,</l>
+
+               <l n="24"> Sola loci facies maesta, silensque placet.</l>
+
+               <l n="25">Non aures cantus, non fila loquacia mulcent;</l>
+
+               <l n="26"> Non oculos formae gratia, flosque rapit.</l>
+
+               <l n="27">Unam te in sylvis, unam in florentibus hortis,</l>
+
+               <l n="28">Per juga, per valles quaero, nec invenio.</l>
+
+               <l n="29">Nec magis Eurydice est Vati quaesita marito,</l>
+
+               <l n="30"> Tartareum quamvis viderit ille canem:</l>
+
+               <l n="31">Nec magis est Cephalo Procris defleta, videnti</l>
+
+               <l n="32"> Deceptae errorem, flagitiumque manus;</l>
+
+               <l n="33">Quam totas ego te noctes, mea vita, diesque</l>
+
+               <l n="34"> Quaero, nec inventam maestus abesse queror.</l>
+
+               <l n="35">Et tamen ante oculos errat tua semper imago,</l>
+
+               <l n="36"> (Quid non fingit amans?) et tua verba sonant.</l>
+
+               <l n="37">Si qua avis in densis, Siren innoxia, lucis</l>
+
+               <l n="38"> Est audita mihi fundere dulce melos,</l>
+
+               <l n="39">Sisto gradum, et similis deceptus imagine vocis</l>
+
+               <l n="40"> Est, inquam, est cantus conjugis ille meae.</l>
+
+               <l n="41">Si quando ad fontes, aut ad vernantia prata,</l>
+
+               <l n="42"> Aut maris ad placidas me tulit error aquas,</l>
+
+               <l n="43">Hic locus est, dico, quem visere saepe solebat.</l>
+
+               <l n="44"> Quae mora (jam sol est ortus) abesse facit?</l>
+
+               <l n="45">Sed jam jam veniet; latet illa forte sub umbra,</l>
+
+               <l n="46"> Aut illi pietas est sua causa morae.</l>
+
+               <l n="47">Causa morae est certe pietas: nisi fallimur, haec est,</l>
+
+               <l n="48"> Fundentem ad superos quae videt hora preces.</l>
+
+               <l n="49">Mox sat ut illusum me liquit amabilis error,</l>
+
+               <l n="50"> Protinus ex oculis bina fluenta cadunt.</l>
+
+               <l n="51">Bina fluenta cadunt, quorum hinc dolor elicit unum,</l>
+
+               <l n="52"> Inde aliud, tanti causa doloris, amor.</l>
+
+               <l n="53">Meque ipsum incuso, quod sim tam stultus, et amens,</l>
+
+               <l n="54"> Et pascam aerumnas crudelitate meas:</l>
+
+               <l n="55">Rursus in errores tamen hos delabor, et hujus</l>
+
+               <l n="56"> Erroris rursus paenitet esse reum.</l>
+
+               <l n="57">Sic pugnant mea vota meis contraria votis,</l>
+
+               <l n="58"> Nec placet, heu! misero quod modo dulce fuit.</l>
+
+               <l n="59">Nec quod sim discors, angit modo; saevius angit</l>
+
+               <l n="60"> Vivere me longos te sine posse dies.</l>
+
+               <l n="61">Ah! ubi sunt voces illae, et mea fortia verba?</l>
+
+               <l n="62"> Ah! ubi, quae verbis debet inesse, fides?</l>
+
+               <l n="63">Me quoque rapturam subito, quae te hora tulisset,</l>
+
+               <l n="64"> Et pariter praedam mortis utrumque fore?</l>
+
+               <l n="65">Ecce tamen vivo, nec post nova cornua Phoebes</l>
+
+               <l n="66"> Vis me maeroris perdere longa potest.</l>
+
+               <l n="67">Heu! quae dura silex, quod inexsuperabile robur,</l>
+
+               <l n="68"> Quod ferrum, et triplex aes mihi pectus obit?</l>
+
+               <l n="69">Vivo equidem, vivo, sed morte est tristior ipsa,</l>
+
+               <l n="70"> Quae sine te, conjux, vita relicta mihi est.</l>
+
+               <l n="71">At tu nunc choreis Natorum immixta tuorum,</l>
+
+               <l n="72"> Qui (prona) facili ad Superos te praeiere (via) gradu,</l>
+
+               <l n="73">Plena Deo frueris, nec, quae tibi parta, bonorum</l>
+
+               <l n="74"> Amittendorum te timor ullus habet.</l>
+
+               <l n="75">Nam tua non tristes pietas te duxit in oras:</l>
+
+               <l n="76"> Debetur sedes non nisi laeta piis.</l>
+
+               <l n="77">Te plaga (credo equidem) summi plaga lucida caeli,</l>
+
+               <l n="78"> Te laeta aeterno vere vireta tenent.</l>
+
+               <l n="79">Ipsum ipsum Auctorem rerum, quem qui videt, ultra</l>
+
+               <l n="80"> Nil habet optandum, jam sine nube vides.</l>
+
+               <l n="81">Usque et ubique vides, at non saturata videndo</l>
+
+               <l n="82"> Illo oculos pascis; pressa sed usque fame es.</l>
+
+               <l n="83">Te vis implet opum, sed non (licet impleat) explet;</l>
+
+               <l n="84"> Excipit unum aliud, subsequiturque bonum.</l>
+
+               <l n="85">Non te humiles curae, non te mortalia tangunt;</l>
+
+               <l n="86"> Prae caelo, et stellis quam tibi sordet humus!</l>
+
+               <l n="87">Sordet humus certe. non sic tamen, ut tua nunquam</l>
+
+               <l n="88"> Ad miserum flectas lumina blanda virum;</l>
+
+               <l n="89">Audire aut flentem fugias, et saucia flentis,</l>
+
+               <l n="90"> Qua licet, admota corda fovere manu;</l>
+
+               <l n="91">Iactatumque diu ventisque undisque vocare</l>
+
+               <l n="92"> Ad laeta Eridani littora stelliferi.</l>
+
+               <l n="93">Quam tua sors felix, quam nostra simillima morti est,</l>
+
+               <l n="94"> Felle ego, tu Divum vesceris ambrosia.</l>
+
+               <l n="95">Non tamen invideo tua gaudia, sed miser opto,</l>
+
+               <l n="96"> Laetitiae consors quam prius esse tuae.</l>
+            </lg>
+         </div>
+      </body>
+   </text>
+</TEI>

data/spec/fixtures/petrov_eleg02_with_internal_error.xml

@@ -0,0 +1,255 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-model href="http://www.stoa.org/epidoc/schema/latest/tei-epidoc.rng" type="application/xml" schematypens="http://relaxng.org/ns/structure/1.0"?>
+<TEI xmlns="http://www.tei-c.org/ns/1.0">
+   <text xml:lang="lat">
+      <body>
+         <div type="edition" subtype="poesis-elegia" n="urn:cts:croala:petrov02.eleg02.lat1">
+            <lg>
+               <l n="1">Siccine dividimur, vita mihi carior Uxor,</l>
+
+               <l n="2">Morte tua nostrum dissociante thorum?</l>
+
+               <l n="3">Siccine me miserum, nunquam relictura, relinquis</l>
+
+               <l n="4">Me miserum, et Natos, pignora cara, tuos?</l>
+
+               <l n="5">Non te Bactra tenent, non me nunc ultima Thule,</l>
+
+               <l n="6">Longius est spatium, nos quod abesse facit.</l>
+
+               <l n="7">Nunc mihi, nunc cantum ventosque, amnesque morantem,</l>
+
+               <l n="8">Nunc cuperem auditas Manibus esse fides.</l>
+
+               <l n="9">Non summo ut faciam quercus descendere ab Aemo;</l>
+
+               <l n="10">Aut stare immotas voce canentis aves.</l>
+
+               <l n="11">Sed carae ut repetam fugientem conjugis Umbram</l>
+
+               <l n="12">Nunc prece, nunc <sic>cythara</sic> consociante preces</l>
+
+               <l n="13">Irem audax (amor ipse facem praeferret eunti)</l>
+
+               <l n="14">Plena tenebrarum per loca, plena metus.</l>
+
+               <l n="15">Atque aliquis miserans maesta de gente silentum,</l>
+
+               <l n="16">Quos olim simili vulnere stravit amor,</l>
+
+               <l n="17">Ostendens procul Elysium, loca laeta piorum,</l>
+
+               <l n="18">„Quam petis, ille tenet“, diceret, „ille locus“.</l>
+
+               <l n="19">Non me, non novies Styx interfusa teneret</l>
+
+               <l n="20">Viminea et novies trajicienda rate.</l>
+
+               <l n="21">Non fera vipereo terreret crine Megaera,</l>
+
+               <l n="22">Majestas Stygii non truculenta Jovis.</l>
+
+               <l n="23">Forsan et ille pius <sic>lacrymas</sic> ad verba precantis</l>
+
+               <l n="24">Funderet, et reditum Conjugis annueret.</l>
+
+               <l n="25">Quid loquor ah! demens? non est revocabilis ultra,</l>
+
+               <l n="26">Transvecta est Stygios quae semel Umbra lacus.</l>
+
+               <l n="27">Non redit ad fontem, quae fontem deserit unda.</l>
+
+               <l n="28">Non reflorescunt, quae cecidere, rosae.</l>
+
+               <l n="29">Ergo, quod est miseris reliquum solumque levamen,</l>
+
+               <l n="30">Quo lever, heu! superest nil nisi flere mihi,</l>
+
+               <l n="31">Nil nisi flere meos, saevissima vulnera, casus,</l>
+
+               <l n="32">Dum cineri uxoris jungar et ipse cinis.</l>
+
+               <l n="33">Vix afflicta Domus geminati insignia luctus</l>
+
+               <l n="34">Exuerat, madidas non bene sicca genas:</l>
+
+               <l n="35">Ecce cadit saevae jam tertia victima Morti</l>
+
+               <l n="36">Uxor, vulneribus victima caesa tribus.</l>
+
+               <l n="37">Non fuit illa tibi communi lege necanda,</l>
+
+               <l n="38">Impia Mors, plagis percutienda fuit.</l>
+
+               <l n="39">Cuncta tui in miseram <sic>consumpsti</sic> tela furoris,</l>
+
+               <l n="40">In sola haeserunt sed tua tela cute.</l>
+
+               <l n="41">Membra malis cessere, animus fuit altior illis,</l>
+
+               <l n="42">Tu fragilis victrix corporis, ille tui.</l>
+
+               <l n="43">Saepe opus igne fuit, ferroque; ullumne dolentis</l>
+
+               <l n="44">Inter eas poenas edidit illa sonum?</l>
+
+               <l n="45">Num viso extimuit ferro, ingemuitve recepto?</l>
+
+               <l n="46">Non patiens gemitum, sed miser ipse dabam.</l>
+
+               <l n="47">Ah! chalybe immiti languentis caesa rigabat</l>
+
+               <l n="48">Membra cruor: <sic>lacrymis</sic> non maduere genae.</l>
+
+               <l n="49">Num, quando monita est, momenta novissima vitae</l>
+
+               <l n="50">Instare, et celeri jam properare gradu?</l>
+
+               <l n="51">Scilicet hos animos illi mens inscia noxae,</l>
+
+               <l n="52">Inque Deum pietas, et proba vita dabat,</l>
+
+               <l n="53">Hos amor in miseros, miserisque intenta juvandis,</l>
+
+               <l n="54">Clausa sibi, reliquis semper aperta manus:</l>
+
+               <l n="55">Et vigil in Natos cura, et studium acre tuendae</l>
+
+               <l n="56">In nivea morum simplicitate domus:</l>
+
+               <l n="57">Et tormenta per haec candentis atrocia ferri,</l>
+
+               <l n="58">Quae toties visa est sustinuisse libens,</l>
+
+               <l n="59">Posthuma vitandi tormenta, piantia Manes,</l>
+
+               <l n="60">Spes prope certa, sibi si qua pianda forent.</l>
+
+               <l n="61">Hos animos etiam illa dabant, quae pectore forti</l>
+
+               <l n="62">Ex gemina amissa vulnera prole tulit.</l>
+
+               <l n="63">O Superi, quantum vidit maeroris amari,</l>
+
+               <l n="64">Et quantum vidit roboris illa dies!</l>
+
+               <l n="65">Quae caruere genae <sic>lacrymis</sic>, et questibus ora?</l>
+
+               <l n="66">Fundere nec <sic>lacrymas</sic> visa, nec illa queri.</l>
+
+               <l n="67">Exuerat Matrem coram, Matrem intus agebat,</l>
+
+               <l n="68">Non se, sed miserum commiserata Virum.</l>
+
+               <l n="69">Dotibus his tantis quid nunc bona corporis addam?</l>
+
+               <l n="70">Quid cultum ingenuis artibus ingenium?</l>
+
+               <l n="71">Quid quam dulce melos manabat ab ore loquentis,</l>
+
+               <l n="72">In quo opifex mellis fecerat ipsa favos?</l>
+
+               <l n="73">Talem fata mihi rapuere, immitia fata,</l>
+
+               <l n="74">Consortem thalami, subsidiumque mei.</l>
+
+               <l n="75">Post duo dilectae crudelia funera prolis,</l>
+
+               <l n="76">Praesto illa, auxilium quae mihi ferret, erat.</l>
+
+               <l n="77">Tota videbatur gens esse superstes in illa</l>
+
+               <l n="78">Nullaque, ea salva, membra recisa Domus.</l>
+
+               <l n="79">Nec me illa omnino miserum solante putabam,</l>
+
+               <l n="80">Illa levamen erat levantis, et illa comes.</l>
+
+               <l n="81">Decrescit siquidem fidas diffusus in aures</l>
+
+               <l n="82">Luctus, ut in rivos secta fit unda minor.</l>
+
+               <l n="83">Adde alios casus, et quae mala plurima passus,</l>
+
+               <l n="84">Insontem invidiae vi superante, fui:</l>
+
+               <l n="85">Nunc mihi fraude mala jus libertatis ademptum,</l>
+
+               <l n="86">Nunc raptae, parcus quas dabat usus, opes.</l>
+
+               <l n="87">Dum tu aderas, uxor, felix dicebar, eramque,</l>
+
+               <l n="88">Nil nocuit telis sors mihi saeva suis.</l>
+
+               <l n="89">Vim mala perdebant in dulci conjugis ore,</l>
+
+               <l n="90">Et quam tu, poterant aspera fata minus.</l>
+
+               <l n="91">Cui querar infelix? cui nunc mea tristia dicam?</l>
+
+               <l n="92">Quae <sic>lacrymas</sic> posthac terget amica manus?</l>
+
+               <l n="93">Cuncta, illa rapta, rapuit manus invida Parcae,</l>
+
+               <l n="94">Unde afflicta malis mens capiebat opem.</l>
+
+               <l n="95">Nec, quia ter denos mecum una exegerit annos</l>
+
+               <l n="96">Est mihi, ceu sero rapta, dolenda minus.</l>
+
+               <l n="97">Ah! magis illa ligant, quorum est diuturnior usus,</l>
+
+               <l n="98">Et plus, quae plus sunt nexa, dirempta dolent.</l>
+
+               <l n="99">Sic tener, ac mollis ramo si ramus adhaesit,</l>
+
+               <l n="100">Disjungas facili vincla novella manu.</l>
+
+               <l n="101">Frangitur annosus, vel vix victusque gemensque</l>
+
+               <l n="102">Alter ab amplexu solvitur alterius.</l>
+
+               <l n="103">Nullaque sat longa est, quae desinit esse, voluptas,</l>
+
+               <l n="104">Et nulla est magno longa in amore mora.</l>
+
+               <l n="105">Longa mora est tali mihi nunc uxore carenti,</l>
+
+               <l n="106">Omnis mense dies longior, hora die est.</l>
+
+               <l n="107">Quid moror ergo miser? cur non invisa relinquo</l>
+
+               <l n="108">Lumina? cur te non, o mea vita, sequor?</l>
+
+               <l n="109">Tu mihi dicebas, sine me nil dulce futurum,</l>
+
+               <l n="110">Non si ipsum flueret nectar in ora, tibi.</l>
+
+               <l n="111">Quo sola ergo fugis, comitem aspernata maritum?</l>
+
+               <l n="112">In rapidos abeunt cur tua dicta Notos?</l>
+
+               <l n="113">Sed non sola fugis: me tecum, ubicumque moraris,</l>
+
+               <l n="114">Qui bene nos junxit, vivere cogit amor.</l>
+
+               <l n="115">Una mei pars tecum abiit, minor altera mecum est,</l>
+
+               <l n="116">Pars abiit melior, plena dolore manet.</l>
+
+               <l n="117">Ast haec ipsa brevi tua post vestigia curret,</l>
+
+               <l n="118">A majore nequit pars procul esse minor.</l>
+
+               <l n="119">O volet illa dies, quae partes uniat ambas,</l>
+
+               <l n="120">Meque addat comitem tempus in omne tibi!</l>
+
+               <l n="121">Mors tua divisit, mea nos conjungat, et urna</l>
+
+               <l n="122">Consociet, thalamus quos sociare nequit.</l>
+            </lg>
+         </div>
+      </body>
+   </text>
+</TEI>

data/spec/lib/llt/segmenter_spec.rb

@@ -1,6 +1,10 @@
 require 'spec_helper'
 
 describe LLT::Segmenter do
+  def load_fixture(filename)
+    File.read(File.expand_path("../../../fixtures/#{filename}", __FILE__))
+  end
+
   let(:segmenter) { LLT::Segmenter.new }
   describe "#segment" do
     it "returns an array of LLT::Sentence elements" do
@@ -102,6 +106,7 @@ describe LLT::Segmenter do
       it "doesn't break with punctuation in element names II" do
         txt = '<grc.test>text.</grc.test> text 2.'
         sentences = segmenter.segment(txt, xml: true)
+        puts sentences
         sentences.should have(2).items
         sentences[0].to_s.should == '<grc.test>text.</grc.test>'
         sentences[1].to_s.should == 'text 2.'
@@ -140,6 +145,19 @@ describe LLT::Segmenter do
         sentences = segmenter.segment(txt, xml: true)
         sentences.should have(1).item
       end
+
+      it "doesn't fall with multiple closing tags at the end" do
+        txt = '<div type="div1" xml:id="c097"> <l>Numen inest vati, vatum mens consona caelo est, </l> <l n="100">Nec certus scit fallere Apollo. </l>  </div>'
+        sentences = segmenter.segment(txt, xml: true)
+        puts sentences
+        sentences.should have(1).item
+      end
+
+      it "doesn't fall with empty tags" do
+        txt = '<div type="div1" xml:id="c097"> <l>Numen inest vati, vatum mens consona caelo est, </l> <l n="100">Nec certus scit fallere Apollo. </l> <milestone unit="page" n="210"/> </div>'
+        sentences = segmenter.segment(txt, xml: true)
+        sentences.should have(1).item
+      end
     end
 
     context "with xml escaped characters" do
@@ -163,6 +181,16 @@ describe LLT::Segmenter do
         sentences.should have(2).item
         sentences[1].to_s.should == 'success.'
       end
+
+      describe "when CGI.unescaping HTML characters" do
+        it "acknowledges &apos; as potentially trailing delimiter" do
+          txt = '&apos;text.&apos; success.'
+          unescaped = CGI.unescapeHTML(txt)
+          sentences = segmenter.segment(unescaped)
+          sentences.should have(2).item
+          sentences[1].to_s.should == 'success.'
+        end
+      end
     end
 
     context "newline (\\n) handling" do
@@ -319,6 +347,36 @@ describe LLT::Segmenter do
       end
     end
 
+    context "with full TEI files" do
+      it "doesn't go into an endless loop when something is wrong" do
+        txt = load_fixture('petrov_eleg01_with_endless_loop.xml')
+        sentences = segmenter.segment(txt, xml: true)
+        sentences.should_not be_empty
+        sentences.should have(60).items
+      end
+
+      it "example II" do
+        txt = load_fixture('petrov_eleg01_with_endless_loop_no_xml_header.xml')
+        sentences = segmenter.segment(txt, xml: true)
+        sentences.should_not be_empty
+        sentences.should have(60).items
+      end
+
+      it "example III" do
+        txt = load_fixture('petrov_eleg01_cleaned.xml')
+        sentences = segmenter.segment(txt, xml: true)
+        sentences.should_not be_empty
+        sentences.should have(60).items
+      end
+
+      it "example IV" do
+        txt = load_fixture('petrov_eleg02_with_internal_error.xml')
+        sentences = segmenter.segment(txt, xml: true)
+        sentences.should_not be_empty
+        sentences.should have(74).items
+      end
+    end
+
     describe "takes an optional keyword argument add_to" do
       class ParagraphDummy
         attr_reader :sentences

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: llt-segmenter
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - Gernot Höflechner, Robert Lichstensteiner, Christof Sirk
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-11 00:00:00.000000000 Z
+date: 2014-08-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -42,16 +42,16 @@ dependencies:
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.14'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.14'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -127,8 +127,13 @@ files:
 - lib/llt/segmenter.rb
 - lib/llt/segmenter/api.rb
 - lib/llt/segmenter/version.rb
+- lib/llt/segmenter/version_info.rb
 - lib/llt/sentence.rb
 - llt-segmenter.gemspec
+- spec/fixtures/petrov_eleg01_cleaned.xml
+- spec/fixtures/petrov_eleg01_with_endless_loop.xml
+- spec/fixtures/petrov_eleg01_with_endless_loop_no_xml_header.xml
+- spec/fixtures/petrov_eleg02_with_internal_error.xml
 - spec/lib/llt/segmenter/api_spec.rb
 - spec/lib/llt/segmenter_spec.rb
 - spec/spec_helper.rb
@@ -157,6 +162,10 @@ signing_key:
 specification_version: 4
 summary: Segments text into sentences
 test_files:
+- spec/fixtures/petrov_eleg01_cleaned.xml
+- spec/fixtures/petrov_eleg01_with_endless_loop.xml
+- spec/fixtures/petrov_eleg01_with_endless_loop_no_xml_header.xml
+- spec/fixtures/petrov_eleg02_with_internal_error.xml
 - spec/lib/llt/segmenter/api_spec.rb
 - spec/lib/llt/segmenter_spec.rb
 - spec/spec_helper.rb