llt-tokenizer 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c59bde34626f221dcf1880325b147dc2673c055f
+  data.tar.gz: f78d95a200b8dac652e0cc9a77363c57514c230d
+SHA512:
+  metadata.gz: e1240191c6edec8d7a942504dccad5eb1aa539644ea3c852f050f6b9cea2b91fb76c6c2a310fa076c17aafdd9a8d6ed5ed14a8e355b8bb8be88faf8ab564f0b8
+  data.tar.gz: 65469d5164e9960c21608bafa694e907ad5ae25d3c281f2fad22b94ebbd6db80cb3789d2a66e4748d7c71021c8dd79194f456a848690bed5363afda60af4f03f

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,7 @@
+language: ruby
+before_script:
+  - 'export JRUBY_OPTS=--2.0'
+rvm:
+  - 2.1.0
+  - 2.0.0
+  - jruby-20mode

data/Gemfile

@@ -0,0 +1,27 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in llt-tokenizer.gemspec
+gemspec
+
+gem 'coveralls', require: false
+
+gem 'llt-core', git: 'git@github.com:latin-language-toolkit/llt-core.git'
+gem 'llt-core_extensions', git: 'git@github.com:latin-language-toolkit/llt-core_extensions.git'
+gem 'llt-constants', git: 'git@github.com:latin-language-toolkit/llt-constants.git'
+gem 'llt-db_handler', git: 'git@github.com:latin-language-toolkit/llt-db_handler.git'
+gem 'llt-helpers', git: 'git@github.com:latin-language-toolkit/llt-helpers.git'
+
+# Dependencies of db_handler
+gem 'llt-core_extensions', git: 'git@github.com:latin-language-toolkit/llt-core_extensions.git'
+gem 'llt-form_builder', git: 'git@github.com:latin-language-toolkit/llt-form_builder.git'
+
+platform :ruby do
+  gem 'pg'
+end
+
+platform :jruby do
+  gem 'activerecord-jdbcpostgresql-adapter'
+  gem 'jruby-httpclient'
+end
+
+gem 'pry'

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 LFDM
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,97 @@
+# LLT::Tokenizer
+
+Flexible service to tokenize Latin texts.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'llt-tokenizer'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install llt-tokenizer
+
+## Usage
+
+The LLT's Tokenizer makes use of stem dictionaries. Refer to [these instructions](http://github.com/latin-language-toolkit/llt-db_handler "llt-db_handler") on how to set one up.
+
+```ruby
+  require 'llt/tokenizer'
+ 
+  t = LLT::Tokenizer.new
+  tokens = t.tokenize('Arma virumque cano.')
+  tokens.map(&:to_s)
+  # => ["Arma", "-que", "virum", "cano", "."]
+```
+
+The Tokenizer takes several options upon creation or a call to #tokenize:
+
+```ruby
+  # shifting determines if enclitics shall be moved to
+  # their functional position
+  t = LLT::Tokenizer.new(shifting: true)
+  tokens = t.tokenize('In eoque arma cano.')
+  tokens.map(&:to_s)
+  # => ["-que", "In", "eo", "arma", "cano", "."]
+  
+  # all options can be passed directly to #tokenize to override
+  # the default options
+  tokens = t.tokenize('In eoque arma cano.', shifting: false)
+  tokens.map(&:to_s)
+  # => ["In", "eo", "-que", "arma", "cano", "."]
+  
+  # enclitics_marker takes a string, which marks up splitted enclitics
+  t = LLT::Tokenizer.new(enclitics_marker: '--', shifting: false)
+  tokens = t.tokenize('Arma virumque cano.')
+  tokens.map(&:to_s)
+  # => ["Arma", "virum", "--que", "cano", "."]
+
+  # indexing determines if each token shall receive a consecutive id
+  tokens = t.tokenize('Arma virumque cano.', indexing: true)
+  tokens.first.id # => 1
+  tokens = t.tokenize('Arma virumque cano.', indexing: false)
+  tokens.first.id # => nil
+
+  # merging enables token merging of lemmata, that often appear with
+  # orthographical inconsistencies
+  tokens = t.tokenize('Quam diu cano?', merging: true)
+  tokens.map(&:to_s)
+  # => ["Quamdiu", "cano", "?"]
+```
+
+The returned items are instances of LLT::Token, which can be marked up
+in a variety of forms:
+
+```ruby
+  t = LLT::Tokenizer.new(shifting: false, indexing: true)
+  tokens = t.tokenize('Arma virumque cano.')
+  tokens.map(&:to_xml)
+  # => ["<w>arma<_w>", "<w>virum<_w>", "<w>-que<_w>", "<w>cano<_w>", "<pc>.<_pc>"]
+```
+
+Standard TEI XML markup is used: w tags for word tokens, pc tags for
+punctuation. The #to_xml method is highly flexible as well, for full
+coverage see _TODO_.
+
+```ruby
+  puts tokens.map { |token| token.to_xml(indexing: true) }
+  # <w n="1">Arma</w>
+  # <w n="2">virum</w>
+  # <w n="3">-que</w>
+  # <w n="4">cano</w>
+  # <pc n="5">.</pc>
+```
+
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/lib/llt/token/filler.rb

@@ -0,0 +1,31 @@
+module LLT
+  class Token
+    class Filler < Token
+      xml_tag 'w'
+
+      def add(type)
+        case type
+        when :name then add_name_form
+        end
+      end
+
+      #def add_name_form
+      #  @possible_forms << PersonaFiller.new(@word)
+      #end
+
+      # cannot hold anything atm, is therefore never really empty
+      def empty?
+        false
+      end
+      alias :no_forms? :empty?
+
+      def set_functions
+        [:filler]
+      end
+
+      def inspect
+        "#{"Filler token".blue}: #{@string}"
+      end
+    end
+  end
+end

data/lib/llt/token/punctuation.rb

@@ -0,0 +1,36 @@
+module LLT
+  class Token
+    class Punctuation < Token
+      xml_tag 'pc'
+
+      attr_accessor :opening, :closing, :other
+
+      def initialize(string, id = nil)
+        super
+        # this is part of an old interface that is mostly unused
+        # some parts remain - find and delete em
+        @opening = false
+        @closing = false
+        @other   = false
+      end
+
+      # cannot hold anything atm, is therefore never really empty
+      def empty?
+        false
+      end
+      alias :no_forms? :empty?
+
+      def set_functions
+        [:punctuation]
+      end
+
+      def punctuation
+        @string
+      end
+
+      def inspect
+        "#{"Punctuation token:".yellow} #{@string}"
+      end
+    end
+  end
+end

data/lib/llt/token/word.rb

@@ -0,0 +1,53 @@
+module LLT
+  class Token
+    class Word < Token
+      xml_tag 'w'
+
+      def word
+        @string
+      end
+
+      def no_forms?
+        @container.empty?
+      end
+
+      def set_functions
+        [:word]
+      end
+
+      def use(i = nil)
+        if i
+          return @container[i - 1]
+        elsif block_given?
+          @container.find { |f| yield(f) }
+        end
+      end
+
+      def inspect
+        "#{"Word token".green}: #{@string}\n" +
+        "\tForms: #{forms_to_s}\n"
+      end
+
+      def forms_to_s
+        # was each_with_index_and_object, which is currently not available
+        @container.each_with_index.each_with_object("") do |(f, i), str|
+          str << enumeration(i) << stripped_form(f)
+          str << delimiter unless f == @container.last
+          str
+        end
+      end
+
+      def stripped_form(form)
+        form.to_s.sub(@string, "").strip
+      end
+
+      def enumeration(i)
+        "#{i}: ".light_yellow
+      end
+
+      def delimiter
+        " | ".cyan
+      end
+    end
+  end
+end

data/lib/llt/token/xml_tag.rb

@@ -0,0 +1,24 @@
+module LLT
+  class Token
+    class XmlTag < Token
+      def set_functions
+        [:xml_tag]
+      end
+
+      # overrides #to_xml from Containable - the tag stays at is it
+      def to_xml(*args)
+        to_s
+      end
+
+      def inspect
+        "#{'XML tag'.blue} #{tag_status}: #{to_s}"
+      end
+
+      private
+
+      def tag_status
+        to_s.match(/\//) ? 'open' : 'close'
+      end
+    end
+  end
+end

data/lib/llt/token.rb

@@ -0,0 +1,51 @@
+require 'llt/core/containable'
+require 'llt/helpers/functions'
+
+module LLT
+  class Token
+    include Core::Containable
+    include Helpers::Functions
+    #include Phonology
+
+    require 'llt/token/word'
+    require 'llt/token/punctuation'
+    require 'llt/token/filler'
+    require 'llt/token/xml_tag'
+
+    attr_reader :functions, :special_roles
+
+    container_alias :forms
+
+    def initialize(string, id = nil)
+      super
+      @functions = set_functions
+    end
+
+    def special_roles
+      @special_roles || []
+    end
+
+    def has_special_role?(role)
+      special_roles.include?(role)
+    end
+
+    def set_special_role(*roles)
+      @special_roles ||= []
+      @special_roles += roles
+    end
+
+    # deprecated
+    def add_form(form)
+      @forms << form
+    end
+
+    # deprecated
+    def add_forms(forms)
+      @forms += forms
+    end
+
+    def use(*args)
+      # hook method, overwritten by Word
+    end
+  end
+end

data/lib/llt/tokenizer/api.rb

@@ -0,0 +1,20 @@
+require 'sinatra/base'
+require 'sinatra/respond_with'
+require 'llt/tokenizer'
+require 'llt/core/api'
+
+class Api < Sinatra::Base
+  register Sinatra::RespondWith
+  helpers LLT::Core::Api::Helpers
+
+  get '/tokenize' do
+    typecast_params!(params)
+    text = extract_text(params)
+    tokenizer = LLT::Tokenizer.new(params)
+    tokens = tokenizer.tokenize(text)
+
+    respond_to do |f|
+      f.xml { to_xml(tokens, params) }
+    end
+  end
+end

data/lib/llt/tokenizer/version.rb

@@ -0,0 +1,5 @@
+module LLT
+  class Tokenizer
+    VERSION = "0.0.1"
+  end
+end

data/lib/llt/tokenizer/worker.rb

@@ -0,0 +1,106 @@
+require 'forwardable'
+
+module LLT
+  class Tokenizer
+    class Worker
+      extend Forwardable
+      include Enumerable
+      include Helpers::Metrical
+
+      def_delegators :@bare_text, :each, :[], :[]=, :insert, :delete_at,
+                                  :each_overlapping_pair, :map!
+
+      # TODO 28.11.13 11:45 by LFDM
+      # Edge cases?
+      # Merge words?
+
+      def initialize(metric_text, marker)
+        @metric_text = metric_text
+        @bare_text     = metric_text.map { |token| wo_meter(token) }
+        @marker        = marker
+        @marked_enclitics = ENCLITICS.map { |e| "#{@marker}#{e}"}
+      end
+
+      def to_a
+        align_metrical_text
+        @metric_text
+      end
+
+      private
+
+      # One ugly method, but we don't want to slow it down even more
+      def align_metrical_text
+        m = ArrayScanner.new(@metric_text)
+        b = ArrayScanner.new(@bare_text)
+        loop do
+          # metric element
+          x = m.scan
+          # bare element
+          y = b.scan
+          no_meter = wo_meter(x)
+
+          # we don't have to do anything if the dequantified metric element
+          # was the same as the bare element - the metric_text was right
+          # at this position
+          unless no_meter == y
+
+            # If the bare element was a marked enclitic, it must have been
+            # shifted. We're looking for the next metric token, that has it
+            # attached and try to find the string index where it starts to
+            # slice it of.
+            # Usually the metric element just scanned (y) will have it, if we
+            # don't find it, a double shift has occured and it should sit right
+            # at the current element of the metric ArrayScanner (m).
+            # The enclitic (sliced of x) has to be inserted one position before.
+            if @marked_enclitics.include?(y)
+              clean_encl_re = /#{y.dup.delete(@marker)}$/
+              unless index = no_meter =~ clean_encl_re
+                x = m.current
+                index = wo_meter(x) =~ clean_encl_re
+              end
+              insert!(slice_encl!(x, index), m.pos - 1)
+
+            # If the dequantified metric element has an enclitic attached, the
+            # option shifting: false must have been given. The enclitic will
+            # follow right after in the @bare_text, we can therefore slice and
+            # insert right in place (the next # scan round will reveal that
+            # enclitic in metric_text == enclitic in bare_text
+            elsif encl = ENCLITICS.find { |e| no_meter.end_with?(e) }
+              index = no_meter =~ /#{encl}$/
+              insert!(slice_encl!(x, index), m.pos)
+
+            # If the bare element has a dot attached, it must have been an
+            # abbreviation.
+            # The . will appear right afterwards in the metric text. We can
+            # delete it and append it to the last scanned metric element (x)
+            #
+            # We need to do the same if merge words were present.
+            # The last metric element was quam, the bare element is quamdiu.
+            # We append if the last metric element + the next metric element
+            # is the same as the bare element.
+            elsif y.end_with?('.') || merged_words_present?(no_meter, y, m)
+              append_from_deleted_index!(x, m.pos)
+            end
+          end
+          break if b.eoa?
+        end
+      end
+
+      def insert!(enclitic, position)
+        @metric_text.insert(position, "#{@marker}#{enclitic}")
+      end
+
+      def slice_encl!(token, index)
+        token.slice!(index..-1)
+      end
+
+      def append_from_deleted_index!(token, index)
+        token << @metric_text.delete_at(index)
+      end
+
+      def merged_words_present?(last_metric, last_bare, metric_arr_scanner)
+        (last_metric + wo_meter(metric_arr_scanner.peek)) == last_bare
+      end
+    end
+  end
+end