opener-pos-tagger-en-es 2.0.2 → 2.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 89157ca3c263b71b58644e6d5affb554ba67bf68
-  data.tar.gz: b61b257b2e6af19487d7c477f6ab9c8573260d5c
+  metadata.gz: cf8e78b3be6d89b98ddc381d07ad70f5e0a454e4
+  data.tar.gz: 4c92415365aacfa409539d9257ecec6749b647a7
 SHA512:
-  metadata.gz: a78849a4f64d09070a073b25f433e345539e7fed6528720aae83113db9ad3565953594051135f7e3e4f41a2957f5dd63048c2ae67a20d3e37a78f1bcc4f05dde
-  data.tar.gz: 2835d0aae655bc7b5d364f97f476acc2786e73a78a48862b705445631c435d30c5e5a48a1e0bbb20f16622c3b59ae10d4b3fbe885b107f90cc9f8e163ab25f8c
+  metadata.gz: 13e994b170f0519cce76392d7e37d989c766796b1d4760127e41103974a9625ce71dece012166ef58c3eae0640f713d115150ea6a31831e3508c3911d11ed163
+  data.tar.gz: c8a2f87ad1be16242ae300d5b61f8b730ab1bcbce55419c84668e7261fba2a0316066f3c336f5a5ea4cfaf9bc32df0b6c0410b0e1e1ff684e49c52aa13eef996

data/LICENSE.txt

@@ -0,0 +1,13 @@
+Copyright 2014 OpeNER Project Consortium
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -2,20 +2,25 @@
 
 # English, Spanish,  Dutch, Italian, French POS Tagger
 
-**Modified to also do Dutch**
-
 This repository contains the source code for the English & Spanish POS tagger of the
 OpeNER project.
 
-English perceptron models have been trained and evaluated using the WSJ
-treebank as explained in K. Toutanova, D. Klein, and C. D. Manning.
-Feature-rich part-of-speech tagging with a cyclic dependency network.  In
-Proceedings of HLT-NAACL’03, 2003. Currently we obtain a performance of 96.48%
-vs 97.24% obtained by Toutanova et al. (2003).
+* English perceptron models have been trained and evaluated using the WSJ 
+  treebank as explained in K. Toutanova, D. Klein, and C. D. Manning. 
+  Feature-rich part-of-speech tagging with a cyclic dependency network.  In 
+  Proceedings of HLT-NAACL’03, 2003. Currently we obtain a performance of 96.87% 
+  vs 97.24% obtained by Toutanova et al. (2003).
+
+* Spanish Maximum Entropy models have been trained and evaluated using the Ancora  
+  corpus; it was randomly divided in 90% for training (450K words) and 10% testing 
+  (50K words), obtaining a performance of 98.88%.
+
+* French Maximum Entropy models trained with the ESTER corpus. 
+
+* Italian Perceptron models trained with the TUT Treebank. 
 
-Spanish Maximum Entropy models have been trained and evaluated using the Ancora 
-corpus; it was randomly divided in 90% for training (440K words) and 10% testing
-(70K words), obtaining a performance of 98.88%.
+* Dutch Perceptron model publicly available at Apache OpenNLP website:
+  (http://opennlp.sourceforge.net/models-1.5/)
 
 ## Requirements
 

data/bin/pos-tagger-en

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 
-require_relative '../lib/opener/pos_taggers/en'
+require_relative '../lib/opener/pos_taggers/en_es'
 
 # STDIN.tty? returns `false` if data is being piped into the current process.
 if STDIN.tty?
@@ -9,5 +9,6 @@ else
   input = STDIN.read
 end
 
-kernel                  = Opener::POSTaggers::EN.new(:args => ARGV)
+kernel = Opener::POSTaggers::EN.new(:args => ARGV)
+
 puts kernel.run(input)

data/bin/pos-tagger-es

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 
-require_relative '../lib/opener/pos_taggers/en'
+require_relative '../lib/opener/pos_taggers/en_es'
 
 # STDIN.tty? returns `false` if data is being piped into the current process.
 if STDIN.tty?
@@ -10,4 +10,5 @@ else
 end
 
 kernel = Opener::POSTaggers::ES.new(:args => ARGV)
+
 puts kernel.run(input)

data/bin/pos-tagger-fr

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 
-require_relative '../lib/opener/pos_taggers/en'
+require_relative '../lib/opener/pos_taggers/en_es'
 
 # STDIN.tty? returns `false` if data is being piped into the current process.
 if STDIN.tty?
@@ -9,5 +9,6 @@ else
   input = STDIN.read
 end
 
-kernel                  = Opener::POSTaggers::FR.new(:args => ARGV)
+kernel = Opener::POSTaggers::FR.new(:args => ARGV)
+
 puts kernel.run(input)

data/bin/pos-tagger-it

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 
-require_relative '../lib/opener/pos_taggers/en'
+require_relative '../lib/opener/pos_taggers/en_es'
 
 # STDIN.tty? returns `false` if data is being piped into the current process.
 if STDIN.tty?
@@ -9,5 +9,6 @@ else
   input = STDIN.read
 end
 
-kernel                  = Opener::POSTaggers::IT.new(:args => ARGV)
+kernel = Opener::POSTaggers::IT.new(:args => ARGV)
+
 puts kernel.run(input)

data/bin/pos-tagger-nl

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 
-require_relative '../lib/opener/pos_taggers/en'
+require_relative '../lib/opener/pos_taggers/en_es'
 
 # STDIN.tty? returns `false` if data is being piped into the current process.
 if STDIN.tty?
@@ -9,5 +9,6 @@ else
   input = STDIN.read
 end
 
-kernel                  = Opener::POSTaggers::NL.new(:args => ARGV)
+kernel = Opener::POSTaggers::NL.new(:args => ARGV)
+
 puts kernel.run(input)

data/lib/opener/pos_taggers/en.rb

@@ -1,118 +1,3 @@
-require 'open3'
-require 'stringio'
-
-require 'java'
-
-require File.expand_path('../../../../core/target/ehu-pos-1.0.jar', __FILE__)
-
-import 'java.io.InputStreamReader'
-import 'ixa.kaflib.KAFDocument'
-import 'ehu.pos.Annotate'
-import 'ehu.pos.Resources'
-import 'ehu.lemmatize.MorfologikLemmatizer'
-#import 'ehu.lemmatize.Dictionary'
-import 'ehu.lemmatize.LemmatizerDispatcher'
-
-require_relative 'en/version'
-
-module Opener
-  module POSTaggers
-    ##
-    # The POS tagger that supports English and Spanish.
-    #
-    # @!attribute [r] args
-    #  @return [Array]
-    # @!attribute [r] options
-    #  @return [Hash]
-    #
-    class EN
-      attr_reader :args, :options
-
-      ##
-      # The default language to use.
-      #
-      # @return [String]
-      #
-      DEFAULT_LANGUAGE = 'en'.freeze
-
-      ##
-      # @param [Hash] options
-      #
-      # @option options [Array] :args The commandline arguments to pass to the
-      #  underlying Python script.
-      #
-      def initialize(options = {})
-        @args    = options.delete(:args) || []
-        @options = options
-      end
-
-      ##
-      # Runs the command and returns the output of STDOUT, STDERR and the
-      # process information.
-      #
-      # @param [String] input The input to tag.
-      # @return [Array]
-      #
-      def run(input)
-        input     = StringIO.new(input) unless input.kind_of?(IO)
-        reader    = InputStreamReader.new(input.to_inputstream)
-        kaf       = KAFDocument.create_from_stream(reader)
-        annotator = Java::ehu.pos.Annotate.new(language)
-
-        kaf.addLinguisticProcessor("terms","ehu-pos-"+language,"now","1.0");
-        annotator.annotatePOSToKAF(kaf, lemmatizer, language)
-
-        return kaf.to_string
-      end
-
-      protected
-
-      ##
-      # Returns the lemmatizer to use.
-      #
-      def lemmatizer
-        return LemmatizerDispatcher.obtainMorfologikLemmatizer(language)
-      end
-
-      ##
-      # @return [String]
-      #
-      def language
-        return options[:language] || DEFAULT_LANGUAGE
-      end
-    end # EN
-
-    class ES < EN
-      ##
-      # @return [String]
-      #
-      def language
-        return 'es'
-      end
-    end # ES
-
-    class NL < EN
-      def language
-        return 'nl'
-      end
-    end # NL
-
-    class IT < EN
-      ##
-      # @return [String]
-      #
-      def language
-        return 'it'
-      end
-    end # IT
-
-    class FR < EN
-      ##
-      # @return [String]
-      #
-      def language
-        return 'fr'
-      end
-    end # FR
-  end # POSTaggers
-end # Opener
+# This file exists for backwards compatibility, it's recommended to require
+# en_es.rb instead.
+require_relative 'en_es'

data/lib/opener/pos_taggers/en_es.rb

@@ -0,0 +1,19 @@
+require 'open3'
+require 'stringio'
+require 'nokogiri'
+
+require File.expand_path('../../../../core/target/ehu-pos-1.0.jar', __FILE__)
+
+# import 'java.io.InputStreamReader'
+# import 'ixa.kaflib.KAFDocument'
+# import 'ehu.pos.Resources'
+# import 'ehu.lemmatize.MorfologikLemmatizer'
+# import 'ehu.lemmatize.LemmatizerDispatcher'
+
+require_relative 'en_es/version'
+require_relative 'en_es/en_es'
+require_relative 'en_es/en'
+require_relative 'en_es/es'
+require_relative 'en_es/nl'
+require_relative 'en_es/it'
+require_relative 'en_es/fr'

data/lib/opener/pos_taggers/en_es/en.rb

@@ -0,0 +1,16 @@
+module Opener
+  module POSTaggers
+    ##
+    # English POS tagger class. This class forces the language to English
+    # regardless of what the KAF document claims the language to be.
+    #
+    class EN < EnEs
+      ##
+      # @see [Opener::POSTaggers::Base#language_from_kaf]
+      #
+      def language_from_kaf(input)
+        return 'en'
+      end
+    end # EN
+  end # POSTaggers
+end # Opener

data/lib/opener/pos_taggers/en_es/en_es.rb

@@ -0,0 +1,95 @@
+module Opener
+  module POSTaggers
+    ##
+    # Base POS tagger class for the various language specific ones such as
+    # {OpeneR::POSTaggers::FR}.
+    #
+    # @!attribute [r] args
+    #  @return [Array]
+    #
+    # @!attribute [r] options
+    #  @return [Hash]
+    #
+    class EnEs
+      attr_reader :args, :options
+
+      ##
+      # The default options to use.
+      #
+      # @return [Hash]
+      #
+      DEFAULT_OPTIONS = {
+        :enable_time => true
+      }
+
+      ##
+      # @param [Hash] options
+      #
+      # @option options [Array] :args
+      #
+      # @option options [TrueClass|FalseClass] :enable_time When set to `true`
+      #  (default) dynamic timestamps will be added.
+      #
+      def initialize(options = {})
+        @args    = options.delete(:args) || []
+        @options = DEFAULT_OPTIONS.merge(options)
+      end
+
+      ##
+      # Runs the command and returns the resulting KAF document.
+      #
+      # @param [String] input The input to tag.
+      # @return [Array]
+      #
+      def run(input)
+        language = language_from_kaf(input)
+        input    = StringIO.new(input)
+
+        reader    = Java::java.io.InputStreamReader.new(input.to_inputstream)
+        kaf       = Java::ixa.kaflib.KAFDocument.create_from_stream(reader)
+        annotator = new_annotator(language)
+
+        annotator.annotatePOSToKAF(kaf, lemmatizer(language), language)
+
+        return kaf.to_string
+      end
+
+      protected
+
+      ##
+      # Creates and configures a new annotator instance.
+      #
+      # @param [String] language
+      # @return [Java::ehy.pos.Annotate]
+      #
+      def new_annotator(language)
+        annotator = Java::ehu.pos.Annotate.new(language)
+
+        annotator.disableTimestamp unless options[:enable_time]
+
+        return annotator
+      end
+
+      ##
+      # Returns the lemmatizer to use.
+      #
+      # @param [String] language
+      #
+      def lemmatizer(language)
+        return Java::ehu.lemmatize.LemmatizerDispatcher.obtainMorfologikLemmatizer(language)
+      end
+
+      ##
+      # Returns the language for the given KAF document.
+      #
+      # @param [String] input
+      # @return [String]
+      #
+      def language_from_kaf(input)
+        document = Nokogiri::XML(input)
+
+        return document.at('KAF').attr('xml:lang')
+      end
+    end # Base
+  end # POSTaggers
+end # Opener

data/lib/opener/pos_taggers/en_es/es.rb

@@ -0,0 +1,16 @@
+module Opener
+  module POSTaggers
+    ##
+    # Spanish POS tagger class. This class forces the language to Spanish
+    # regardless of what the KAF document claims the language to be.
+    #
+    class ES < EnEs
+      ##
+      # @see [Opener::POSTaggers::Base#language_from_kaf]
+      #
+      def language_from_kaf(input)
+        return 'es'
+      end
+    end # ES
+  end # POSTaggers
+end # Opener

data/lib/opener/pos_taggers/en_es/fr.rb

@@ -0,0 +1,16 @@
+module Opener
+  module POSTaggers
+    ##
+    # French POS tagger class. This class forces the language to French
+    # regardless of what the KAF document claims the language to be.
+    #
+    class FR < EnEs
+      ##
+      # @see [Opener::POSTaggers::Base#language_from_kaf]
+      #
+      def language_from_kaf(input)
+        return 'fr'
+      end
+    end # FR
+  end # POSTaggers
+end # Opener

data/lib/opener/pos_taggers/en_es/it.rb

@@ -0,0 +1,16 @@
+module Opener
+  module POSTaggers
+    ##
+    # Italian POS tagger class. This class forces the language to Italian
+    # regardless of what the KAF document claims the language to be.
+    #
+    class IT < EnEs
+      ##
+      # @see [Opener::POSTaggers::Base#language_from_kaf]
+      #
+      def language_from_kaf(input)
+        return 'it'
+      end
+    end # IT
+  end # POSTaggers
+end # Opener

data/lib/opener/pos_taggers/en_es/nl.rb

@@ -0,0 +1,16 @@
+module Opener
+  module POSTaggers
+    ##
+    # Dutch POS tagger class. This class forces the language to Dutch
+    # regardless of what the KAF document claims the language to be.
+    #
+    class NL < EnEs
+      ##
+      # @see [Opener::POSTaggers::Base#language_from_kaf]
+      #
+      def language_from_kaf(input)
+        return 'nl'
+      end
+    end # NL
+  end # POSTaggers
+end # Opener

data/lib/opener/pos_taggers/{en → en_es}/version.rb

@@ -1,7 +1,7 @@
 module Opener
   module POSTaggers
-    class EN
-      VERSION = "2.0.2"
+    class EnEs
+      VERSION = "2.0.3"
     end
   end
 end

data/opener-pos-tagger-en-es.gemspec

@@ -1,25 +1,30 @@
-require File.expand_path('../lib/opener/pos_taggers/en/version', __FILE__)
+require File.expand_path('../lib/opener/pos_taggers/en_es/version', __FILE__)
 
 Gem::Specification.new do |gem|
   gem.name          = "opener-pos-tagger-en-es"
-  gem.version       = Opener::POSTaggers::EN::VERSION
+  gem.version       = Opener::POSTaggers::EnEs::VERSION
   gem.authors       = ["development@olery.com"]
   gem.summary       = 'POS tagging for English and Spanish'
   gem.description   = gem.summary
   gem.homepage      = "http://opener-project.github.com/"
   gem.has_rdoc      = "yard"
 
+  gem.license = 'Apache 2.0'
+
   gem.required_ruby_version = ">= 1.9.2"
 
   gem.files = Dir.glob([
     'core/target/ehu-pos-*.jar',
     'lib/**/*',
     '*.gemspec',
-    'README.md'
+    'README.md',
+    'LICENSE.txt'
   ]).select { |file| File.file?(file) }
 
   gem.executables = Dir.glob('bin/*').map { |file| File.basename(file) }
 
+  gem.add_dependency 'nokogiri'
+
   gem.add_development_dependency 'rspec', '~> 3.0'
   gem.add_development_dependency 'cucumber'
   gem.add_development_dependency 'rake'

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: opener-pos-tagger-en-es
 version: !ruby/object:Gem::Version
-  version: 2.0.2
+  version: 2.0.3
 platform: ruby
 authors:
 - development@olery.com
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2014-06-17 00:00:00.000000000 Z
+date: 2014-06-30 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :runtime
 - !ruby/object:Gem::Dependency
   name: rspec
   version_requirements: !ruby/object:Gem::Requirement
@@ -79,16 +93,25 @@ extra_rdoc_files: []
 files:
 - core/target/ehu-pos-1.0.jar
 - lib/opener/pos_taggers/en.rb
-- lib/opener/pos_taggers/en/version.rb
+- lib/opener/pos_taggers/en_es.rb
+- lib/opener/pos_taggers/en_es/nl.rb
+- lib/opener/pos_taggers/en_es/fr.rb
+- lib/opener/pos_taggers/en_es/en.rb
+- lib/opener/pos_taggers/en_es/it.rb
+- lib/opener/pos_taggers/en_es/version.rb
+- lib/opener/pos_taggers/en_es/en_es.rb
+- lib/opener/pos_taggers/en_es/es.rb
 - opener-pos-tagger-en-es.gemspec
 - README.md
+- LICENSE.txt
 - bin/pos-tagger-nl
 - bin/pos-tagger-en
 - bin/pos-tagger-fr
 - bin/pos-tagger-es
 - bin/pos-tagger-it
 homepage: http://opener-project.github.com/
-licenses: []
+licenses:
+- Apache 2.0
 metadata: {}
 post_install_message:
 rdoc_options: []