llt-tokenizer 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b0a66574ca8827b73d99ceb5c8dd59a7e1b12040
-  data.tar.gz: 998792856ca6096a28eb2def54349bd973f30176
+  metadata.gz: 75d2abc5e72328a1b4ef2f224931c0656572e71d
+  data.tar.gz: b1b155c05e45b87cfec7f0eb4e1535d9f72783b5
 SHA512:
-  metadata.gz: 26cc6ceb5702552ba927e4a3c2c38548a083915598998dc9bc77beab412b9c65321fc6da88553d8ade4ac409024aecae55f901024776d6aea65b1613f340200e
-  data.tar.gz: e6ed6a22cc74fd58305e043109febe235f948a1b2c769415fd1fa471655b87bab924b5458541500e68957a3b7c5c200377ecd6d89831c19d346ce887806e7b6f
+  metadata.gz: 1a7f8fbd9be93c7053fe601243d6ea4c3a603b0c1643a5f102727844ae06c6083acbeb805bdf74e870c5ceb9c92e9dfdd796b8b201a434f50ef50d532fb65a93
+  data.tar.gz: 37d6ab4e7e39a30b6165e61e0a4d341e4620de0a90521bb17ac8ce141356dbe2dc3a952d39d57dd1e4c90eba224f9077b2bc12ecd5d818c8c9372734c91593fa

data/Gemfile

@@ -12,7 +12,6 @@ gem 'llt-db_handler', git: 'git@github.com:latin-language-toolkit/llt-db_handler
 gem 'llt-helpers', git: 'git@github.com:latin-language-toolkit/llt-helpers.git'
 
 # Dependencies of db_handler
-gem 'llt-core_extensions', git: 'git@github.com:latin-language-toolkit/llt-core_extensions.git'
 gem 'llt-form_builder', git: 'git@github.com:latin-language-toolkit/llt-form_builder.git'
 
 platform :ruby do

data/README.md

@@ -51,6 +51,11 @@ The Tokenizer takes several options upon creation or a call to #tokenize:
   tokens.map(&:to_s)
   # => ["Arma", "virum", "--que", "cano", "."]
 
+  # splitting of enclitics can be disabled altogether
+  tokens = t.tokenize('Arma virumque cano.', splitting: false)
+  tokens.map(&:to_s)
+  # => ["Arma", "virumque", "cano", "."]
+
   # indexing determines if each token shall receive a consecutive id
   tokens = t.tokenize('Arma virumque cano.', indexing: true)
   tokens.first.id # => 1

data/lib/llt/token/punctuation.rb

@@ -1,14 +1,19 @@
+require 'xml_escape'
+
 module LLT
   class Token
     class Punctuation < Token
       xml_tag 'pc'
 
+      include XmlEscape
+
       attr_accessor :opening, :closing, :other
 
       def initialize(string, id = nil)
         super
         # this is part of an old interface that is mostly unused
         # some parts remain - find and delete em
+        @string = xml_decode(string)
         @opening = false
         @closing = false
         @other   = false
@@ -31,6 +36,10 @@ module LLT
       def inspect
         "#{"Punctuation token:".yellow} #{@string}"
       end
+
+      def as_xml
+        xml_encode(@string)
+      end
     end
   end
 end

data/lib/llt/tokenizer.rb

@@ -24,6 +24,8 @@ module LLT
         enclitics_marker: '-',
         merging: true,
         indexing: true,
+        splitting: true,
+        xml: false,
       }
     end
 
@@ -34,7 +36,7 @@ module LLT
       setup(text, options)
 
       find_abbreviations_and_join_strings
-      split_enklitika_and_change_their_position
+      split_enklitika_and_change_their_position if @splitting
       merge_what_needs_merging if @merging # quam diu => quamdiu
       tokens = create_tokens
 
@@ -43,17 +45,20 @@ module LLT
     end
 
     def setup(text, options = {}, worker = [])
-      @text   = text
+      @text = text
       evaluate_metrical_presence(@text)
       @enclitics_marker = parse_option(:enclitics_marker, options)
       @merging          = parse_option(:merging, options)
       @shifting         = parse_option(:shifting, options)
+      @splitting        = parse_option(:splitting, options)
       @indexing         = parse_option(:indexing, options)
+      @xml              = parse_option(:xml, options)
       @worker = setup_worker(worker)
       @shift_range = shift_range(@shifting)
     end
 
-    PUNCTUATION = /([\.\?,!;\-:"'”\(\)\[\]†]|<\/?.+?>)\1*/
+    PUNCTUATION = /&(?:amp|quot|apos|lt|gt);|([\.\?,!;\-:"'”&\(\)\[\]†<>])\1*/
+    XML_TAG = /<\/?.+?>/
 
     # This is here for two reasons:
     #   1) easier test setup, when a preliminary result shall be further evaluated
@@ -64,16 +69,15 @@ module LLT
     #      if it's needed - which should perform better, when there
     #      are no metrics involved (the default case)
     def setup_worker(worker)
-      if worker.any?
-        worker
+      return worker if worker.any?
+
+      elements = split_and_space_text
+      put_xml_attributes_back_together(elements) if @xml
+
+      if metrical?
+        Worker.new(elements, @enclitics_marker)
       else
-        elements = @text.gsub(PUNCTUATION, ' \0 ').split
-        put_xml_attributes_back_together(elements)
-        if metrical?
-          Worker.new(elements, @enclitics_marker)
-        else
-          elements
-        end
+        elements
       end
     end
 
@@ -81,19 +85,23 @@ module LLT
       shifting_enabled ? 0 : 1
     end
 
+    def split_and_space_text
+      regex = @xml ? Regexp.union(XML_TAG, PUNCTUATION) : PUNCTUATION
+      @text.gsub(regex, ' \0 ').split
+    end
+
     def put_xml_attributes_back_together(elements)
-      # elements could be like this
-      # ['<tag', 'attr1="val"', 'attr1="val>']
-      # and we want the complete xml tag back together
       as = ArrayScanner.new(elements)
       loop do
-        last = as.look_behind
-        if last && last.start_with?('<') &! last.end_with?('>')
-          if as.current.match(/\w+=".*"$|>/)
+        last = as.look_behind.to_s # catch nil
+        if open_xml_tag?(last)
+          number_of_xml_elements = as.peek_until do |el|
+            el.end_with?('>')
+          end.size + 1
+
+          number_of_xml_elements.times do
             last << ' ' << as.current
             elements.delete_at(as.pos)
-            # we don't need to forward, as we delete an element anyway
-            next
           end
         else
           as.forward(1)
@@ -102,12 +110,18 @@ module LLT
       end
     end
 
+    def open_xml_tag?(str)
+      str.start_with?('<') &! str.end_with?('>')
+    end
+
 
   ######################
 
     # covers abbreviated Roman praenomen like Ti. in Ti. Claudius Nero
     # covers Roman date expression like a. d. V. Kal. Apr.
     ABBREVIATIONS = /^(#{ALL_ABBRS_PIPED})$/
+    # covers a list of words which are abbreviated with a ' like satin' for satisne
+    APOSTROPHE_WORDS = /^(#{APOSTROPHES_PIPED})$/
 
     # %w{ Atque M . Cicero mittit } to %w{ Atque M. Cicero mittit }
 
@@ -115,7 +129,7 @@ module LLT
       arr = []
       @worker.each_with_index do |e, i|
         n = @worker[i + 1]
-        if e =~ ABBREVIATIONS && n == "."
+        if (n == '.' && e =~ ABBREVIATIONS) || (n == "'" && e =~ APOSTROPHE_WORDS)
           @worker[i + 1] = n.prepend(e)
           arr << (i - arr.size)
         end
@@ -324,7 +338,7 @@ module LLT
 
     ABBR_NAME_WITH_DOT       = /^(#{NAMES_PIPED})\.$/
     ROMAN_DATE_EXPR_WITH_DOT = /^(#{DATES_PIPED})\.$/
-    PUNCT_ITSELF             = Regexp.new(PUNCTUATION.source + '$')
+    PUNCT_ITSELF             = Regexp.new("^(?:#{PUNCTUATION.source})$")
     XML_TAG                  = /<\/?.+?>/
 
     def create_tokens

data/lib/llt/tokenizer/version.rb

@@ -1,5 +1,5 @@
 module LLT
   class Tokenizer
-    VERSION = "0.0.2"
+    VERSION = "0.0.3"
   end
 end

data/spec/lib/llt/token/punctuation_spec.rb

@@ -0,0 +1,17 @@
+require 'spec_helper'
+
+describe LLT::Token::Punctuation do
+  describe "#initialize" do
+    it "normalizes escaped xml characters" do
+      punct = LLT::Token::Punctuation.new('&')
+      punct.to_s.should == '&'
+    end
+  end
+
+  describe "#as_xml" do
+    it "overrides LLT::Core::Containable#as_xml to use xml encodings" do
+      punct = LLT::Token::Punctuation.new('&')
+      punct.as_xml.should == "&"
+    end
+  end
+end

data/spec/lib/llt/tokenizer_spec.rb

@@ -18,10 +18,16 @@ describe LLT::Tokenizer do
       end
 
       it "handles all kinds of parens as well as cruces" do
-        txt = "Marcus (et Claudius) †amici† [sunt]."
+        txt = "<Marcus> (et Claudius) †amici† [sunt]."
         tokens = tokenizer.tokenize(txt)
-        tokens.should have(12).items
-        tokens.map(&:to_s).should == %w{ Marcus ( et Claudius ) † amici † [ sunt ] . }
+        tokens.should have(14).items
+        tokens.map(&:to_s).should == %w{ < Marcus > ( et Claudius ) † amici † [ sunt ] . }
+      end
+
+      it "handles escaped xml characters" do
+        txt = "&amp; &quot; &apos; &gt; &lt; ;"
+        tokens = tokenizer.tokenize(txt)
+        tokens.should have(6).items
       end
 
       describe "takes an optional keyword argument add_to" do
@@ -110,19 +116,11 @@ describe LLT::Tokenizer do
             end
           end
         end
-
-        context "with embedded xml tags" do
-          it "doesn't break" do
-            txt = '<grc>text text</grc>'
-            tokens = tokenizer.tokenize(txt)
-            tokens.should have(4).items
-          end
-        end
       end
     end
 
     describe "#find_abbreviations_and_join_strings" do
-      describe "should bring back abbreviation dots" do
+      describe "should bring back abbreviation dots and apostrophes" do
         it "with names" do
           tokenizer.setup("", {}, %w{ Atque Sex . et M . Cicero . })
           tokenizer.find_abbreviations_and_join_strings
@@ -134,6 +132,12 @@ describe LLT::Tokenizer do
           tokenizer.find_abbreviations_and_join_strings
           tokenizer.preliminary.should == %w{ a. d. V Kal. Apr. }
         end
+
+        it "with apostrophe" do
+          tokenizer.setup("", {}, %w{ ' Apostrophi ' sunt : po ' min ' vin ' tun' scin ' potin ' satin ' })
+          tokenizer.find_abbreviations_and_join_strings
+          tokenizer.preliminary.should == %w{ ' Apostrophi ' sunt : po' min' vin' tun' scin' potin' satin' }
+        end
       end
     end
 
@@ -252,10 +256,10 @@ describe LLT::Tokenizer do
       end
 
       examples = {
-        "Word"     => %w{ ita Marcus quoque -que },
+        "Word"     => %w{ ita Marcus quoque -que po' },
         "Filler"   => %w{ M. Sex. App. Ap. Tib. Ti. C. a. d. Kal. Ian. }, #I XI MMC }
         "XmlTag"   => %w{ <grc> </grc> },
-        "Punctuation" => %w{ , . ! ? † ( ) [ ] ... -- ” " ' }
+        "Punctuation" => %w{ , . ! ? † ( ) [ ] ... -- ” " ' & < > &amp; &lt; &gt; &apos; &quot; }
       }
 
       examples.each do |klass, elements|
@@ -278,37 +282,11 @@ describe LLT::Tokenizer do
       tokens.map(&:id).should == [1, 2]
     end
 
-    it "can be disabled" do
+    it "id's can be disabled" do
       txt = 'Cano.'
       tokens = tokenizer.tokenize(txt, indexing: false)
       tokens.map(&:id).should == [nil, nil]
     end
-
-    it "doesn't count plain xml tags" do
-      txt = '<grc>text text</grc>'
-      tokens = tokenizer.tokenize(txt)
-      tokens.map(&:id).should == [nil, 1, 2, nil]
-    end
-
-    it "doesn't count xml tags when they come with attributes" do
-      txt = '<foreign lang="lat">Graeca</foreign> lingua est.'
-      tokens = tokenizer.tokenize(txt).map(&:to_s)
-      res = ['<foreign lang="lat">', 'Graeca', '</foreign>', 'lingua', 'est', '.']
-      tokens.should == res
-    end
-
-    it "handles nested xml as well" do
-      txt = '<l n="70"><foreign lang="lat">Graeca lingua est.</foreign></l>'
-      tokens = tokenizer.tokenize(txt).map(&:to_s)
-      res = ['<l n="70">', '<foreign lang="lat">', 'Graeca', 'lingua', 'est', '.', '</foreign>', '</l>']
-      tokens.should == res
-    end
-
-    it "handles text with broken off xml tags (the rest will e.g. be in another sentence)" do
-      txt = "<lg org=\"uniform\" sample=\"complete\"><l>quem vocet divum populus ruentis</l><l>imperi rebus?"
-      tokens = tokenizer.tokenize(txt)
-      tokens.should have(12).items
-    end
   end
 
   context "with options" do
@@ -346,11 +324,68 @@ describe LLT::Tokenizer do
           tokens.should == %w{ quam diu cano ? }
         end
       end
+
+      context "with disabled splitting" do
+        it "doesn't split enclitics" do
+          txt = 'arma virumque cano.'
+          opts = { splitting: false }
+          tokens = tokenizer.tokenize(txt, opts).map(&:to_s)
+          tokens.should == %w{ arma virumque cano . }
+        end
+      end
+
+      context "with xml handling enabled" do
+        let(:xml_tokenizer) { LLT::Tokenizer.new(db: stub_db, xml: true) }
+
+        it "doesn't break when xml is embedded" do
+          txt = '<grc>text text</grc>'
+          tokens = xml_tokenizer.tokenize(txt)
+          tokens.should have(4).items
+        end
+
+        it "doesn't count plain xml tags" do
+          txt = '<grc>text text</grc>'
+          tokens = xml_tokenizer.tokenize(txt)
+          tokens.map(&:id).should == [nil, 1, 2, nil]
+        end
+
+        it "doesn't count xml tags when they come with attributes" do
+          txt = '<foreign lang="lat">Graeca</foreign> lingua est.'
+          tokens = xml_tokenizer.tokenize(txt).map(&:to_s)
+          res = ['<foreign lang="lat">', 'Graeca', '</foreign>', 'lingua', 'est', '.']
+          tokens.should == res
+        end
+
+        it "handles nested xml as well" do
+          txt = '<l n="70"><foreign lang="lat">Graeca lingua est.</foreign></l>'
+          tokens = xml_tokenizer.tokenize(txt).map(&:to_s)
+          res = ['<l n="70">', '<foreign lang="lat">', 'Graeca', 'lingua', 'est', '.', '</foreign>', '</l>']
+          tokens.should == res
+        end
+
+        it "handles text with broken off xml tags (the rest will e.g. be in another sentence)" do
+          txt = "<lg org=\"uniform\" sample=\"complete\"><l>quem vocet divum populus ruentis</l><l>imperi rebus?"
+          tokens = xml_tokenizer.tokenize(txt)
+          tokens.should have(12).items
+        end
+
+        it "doesn't fall with spaces inside of xml attributes" do
+          txt = '<test>veni vidi <bad att="a a a">vici</bad></test>'
+          tokens = xml_tokenizer.tokenize(txt)
+          tokens.should have(7).items
+        end
+
+        it "expects all text chevrons to be escaped, otherwise they are xml tags!" do
+          txt = '<test>&lt;veni&gt;</test>'
+          tokens = xml_tokenizer.tokenize(txt)
+          tokens.should have(5).item
+        end
+      end
     end
   end
 
   context "with options on instance creation" do
-    it "a new instance can receive options, which it will use as it's defaults" do
+    it "a new instance can receive options, which it will use as its defaults" do
       custom_tok = LLT::Tokenizer.new(db: stub_db,
                                       shifting: false,
                                       enclitics_marker: '')

metadata

@@ -1,141 +1,141 @@
 --- !ruby/object:Gem::Specification
 name: llt-tokenizer
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - LFDM
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-09 00:00:00.000000000 Z
+date: 2014-01-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '1.3'
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
-  prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: rake
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: rspec
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: simplecov
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.7'
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.7'
-  prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: array_scanner
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.7'
+- !ruby/object:Gem::Dependency
+  name: array_scanner
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: llt-core
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: llt-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: llt-core_extensions
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: llt-core_extensions
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: llt-db_handler
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: llt-db_handler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: llt-helpers
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: llt-helpers
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
   type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: LLT's Tokenizer
 email:
 - 1986gh@gmail.com
@@ -143,9 +143,9 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .rspec
-- .travis.yml
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -160,6 +160,7 @@ files:
 - lib/llt/tokenizer/version.rb
 - lib/llt/tokenizer/worker.rb
 - llt-tokenizer.gemspec
+- spec/lib/llt/token/punctuation_spec.rb
 - spec/lib/llt/tokenizer/api_spec.rb
 - spec/lib/llt/tokenizer_spec.rb
 - spec/spec_helper.rb
@@ -168,27 +169,28 @@ homepage: ''
 licenses:
 - MIT
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.1.9
-signing_key:
+rubyforge_project: 
+rubygems_version: 2.2.0
+signing_key: 
 specification_version: 4
 summary: Breaks latin sentences into tokens
 test_files:
+- spec/lib/llt/token/punctuation_spec.rb
 - spec/lib/llt/tokenizer/api_spec.rb
 - spec/lib/llt/tokenizer_spec.rb
 - spec/spec_helper.rb