opener-pos-tagger-en-es 2.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 057739e4c1a1282039f21cfc81467a5193578825
+  data.tar.gz: 22a50f89e3f63158ccabd9471d8dea434a11adbc
+SHA512:
+  metadata.gz: 1736f4efcf334f7ff2c835e68df764c96af65e0d688105b313f6b31c06df9591db231984339c50b92a7c64d08e4a8811328eb62ec8b9eec340e7d62e85549030
+  data.tar.gz: 4813bf565fe3480ef1adb04faaee0c766fa4b16058cbdc6782fca5ff3d80fc24e7fd2bcf8e4c4b348e95237e2e413cabb1e1a78e4cda906dc26e77c58f8ab4c6

data/README.md

@@ -0,0 +1,79 @@
+[![Build Status](https://drone.io/github.com/opener-project/pos-tagger-en-es/status.png)](https://drone.io/github.com/opener-project/pos-tagger-en-es/latest)
+
+# English, Spanish,  Dutch, Italian, French POS Tagger
+
+**Modified to also do Dutch**
+
+This repository contains the source code for the English & Spanish POS tagger of the
+OpeNER project.
+
+English perceptron models have been trained and evaluated using the WSJ
+treebank as explained in K. Toutanova, D. Klein, and C. D. Manning.
+Feature-rich part-of-speech tagging with a cyclic dependency network.  In
+Proceedings of HLT-NAACL’03, 2003. Currently we obtain a performance of 96.48%
+vs 97.24% obtained by Toutanova et al. (2003).
+
+Spanish Maximum Entropy models have been trained and evaluated using the Ancora 
+corpus; it was randomly divided in 90% for training (440K words) and 10% testing
+(70K words), obtaining a performance of 98.88%.
+
+## Requirements
+
+* Java 1.7 or newer
+* Ruby 1.9.2 or newer
+* Maven
+* Bundler
+
+## Installation
+
+Using RubyGems:
+
+    gem install opener-pos-tagger-en-es
+
+Using Bundler:
+
+    gem 'opener-pos-tagger-en-es',
+      :git    => 'git@github.com/opener-project/pos-tagger-en-es.git',
+      :branch => 'master'
+
+Using specific install:
+
+    gem install specific_install
+    gem specific_install opener-pos-tagger-en-es \
+        -l https://github.com/opener-project/pos-tagger-en-es.git
+
+## Usage
+
+    cat some_input_file.kaf | pos-tagger-en-es
+
+## Contributing
+
+First make sure all the required dependencies are installed:
+
+    bundle install
+
+Then compile the required Java code:
+
+    bundle exec rake java:compile
+
+For this you'll need to have Java 1.7 and Maven installed. These requirements
+are verified for you before the Rake task calls Maven.
+
+## Testing
+
+To run the tests (which are powered by Cucumber), simply run the following:
+
+    bundle exec rake
+
+This will take care of verifying the requirements, installing the required Java
+packages and running the tests.
+
+For more information on the available Rake tasks run the following:
+
+    bundle exec rake -T
+
+## Structure
+
+This repository comes in two parts: a collection of Java source files and Ruby
+source files. The Java code can be found in the `core/` directory, everything
+else will be Ruby source code.

data/bin/pos-tagger-en

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/opener/pos_taggers/en'
+
+# STDIN.tty? returns `false` if data is being piped into the current process.
+if STDIN.tty?
+  input = nil
+else
+  input = STDIN.read
+end
+
+kernel                  = Opener::POSTaggers::EN.new(:args => ARGV)
+puts kernel.run(input)

data/bin/pos-tagger-es

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/opener/pos_taggers/en'
+
+# STDIN.tty? returns `false` if data is being piped into the current process.
+if STDIN.tty?
+  input = nil
+else
+  input = STDIN.read
+end
+
+kernel = Opener::POSTaggers::ES.new(:args => ARGV)
+puts kernel.run(input)

data/bin/pos-tagger-fr

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/opener/pos_taggers/en'
+
+# STDIN.tty? returns `false` if data is being piped into the current process.
+if STDIN.tty?
+  input = nil
+else
+  input = STDIN.read
+end
+
+kernel                  = Opener::POSTaggers::FR.new(:args => ARGV)
+puts kernel.run(input)

data/bin/pos-tagger-it

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/opener/pos_taggers/en'
+
+# STDIN.tty? returns `false` if data is being piped into the current process.
+if STDIN.tty?
+  input = nil
+else
+  input = STDIN.read
+end
+
+kernel                  = Opener::POSTaggers::IT.new(:args => ARGV)
+puts kernel.run(input)

data/bin/pos-tagger-nl

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/opener/pos_taggers/en'
+
+# STDIN.tty? returns `false` if data is being piped into the current process.
+if STDIN.tty?
+  input = nil
+else
+  input = STDIN.read
+end
+
+kernel                  = Opener::POSTaggers::NL.new(:args => ARGV)
+puts kernel.run(input)

data/lib/opener/pos_taggers/en/version.rb

@@ -0,0 +1,7 @@
+module Opener
+  module POSTaggers
+    class EN
+      VERSION = "2.0.0"
+    end
+  end
+end

data/lib/opener/pos_taggers/en.rb

@@ -0,0 +1,118 @@
+require 'open3'
+require 'stringio'
+
+require 'java'
+
+require File.expand_path('../../../../core/target/ehu-pos-1.0.jar', __FILE__)
+
+import 'java.io.InputStreamReader'
+import 'ixa.kaflib.KAFDocument'
+import 'ehu.pos.Annotate'
+import 'ehu.pos.Resources'
+import 'ehu.lemmatize.MorfologikLemmatizer'
+import 'ehu.lemmatize.Dictionary'
+
+require_relative 'en/version'
+
+module Opener
+  module POSTaggers
+    ##
+    # The POS tagger that supports English and Spanish.
+    #
+    # @!attribute [r] args
+    #  @return [Array]
+    # @!attribute [r] options
+    #  @return [Hash]
+    #
+    class EN
+      attr_reader :args, :options
+
+      ##
+      # The default language to use.
+      #
+      # @return [String]
+      #
+      DEFAULT_LANGUAGE = 'en'.freeze
+
+      ##
+      # @param [Hash] options
+      #
+      # @option options [Array] :args The commandline arguments to pass to the
+      #  underlying Python script.
+      #
+      def initialize(options = {})
+        @args    = options.delete(:args) || []
+        @options = options
+      end
+
+      ##
+      # Runs the command and returns the output of STDOUT, STDERR and the
+      # process information.
+      #
+      # @param [String] input The input to tag.
+      # @return [Array]
+      #
+      def run(input)
+        input     = StringIO.new(input) unless input.kind_of?(IO)
+        reader    = InputStreamReader.new(input.to_inputstream)
+        kaf       = KAFDocument.create_from_stream(reader)
+        annotator = Java::ehu.pos.Annotate.new(language)
+
+        kaf.addLinguisticProcessor("terms","ehu-pos-"+language,"now","1.0");
+        annotator.annotatePOSToKAF(kaf, lemmatizer, language)
+
+        return kaf.to_string
+      end
+
+      protected
+
+      def dictionary
+        Resources.new.getBinaryDict(language)
+      end
+
+      def lemmatizer
+        MorfologikLemmatizer.new(dictionary)
+      end
+
+      ##
+      # @return [String]
+      #
+      def language
+        return options[:language] || DEFAULT_LANGUAGE
+      end
+    end # EN
+
+    class ES < EN
+      ##
+      # @return [String]
+      #
+      def language
+        return 'es'
+      end
+    end # ES
+
+    class NL < EN
+      def language
+        return 'nl'
+      end
+    end # NL
+    
+    class IT < EN
+      ##
+      # @return [String]
+      #
+      def language
+        return 'it'
+      end
+    end # IT
+    
+    class FR < EN
+      ##
+      # @return [String]
+      #
+      def language
+        return 'fr'
+      end
+    end # FR
+  end # POSTaggers
+end # Opener

data/opener-pos-tagger-en-es.gemspec

@@ -0,0 +1,27 @@
+require File.expand_path('../lib/opener/pos_taggers/en/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name          = "opener-pos-tagger-en-es"
+  gem.version       = Opener::POSTaggers::EN::VERSION
+  gem.authors       = ["development@olery.com"]
+  gem.summary       = "POS tagging for english, spanish, dutch, italian and french"
+  gem.description   = gem.summary
+  gem.homepage      = "http://opener-project.github.com/"
+  gem.has_rdoc      = "yard"
+  gem.required_ruby_version = ">= 1.9.2"
+
+  gem.files = Dir.glob([
+    'core/target/ehu-pos-*.jar',
+    'lib/**/*',
+    '*.gemspec',
+    'README.md'
+  ]).select { |file| File.file?(file) }
+
+  gem.executables = Dir.glob('bin/*').map { |file| File.basename(file) }
+
+  gem.add_dependency 'opener-build-tools'
+
+  gem.add_development_dependency 'rspec'
+  gem.add_development_dependency 'cucumber'
+  gem.add_development_dependency 'rake'
+end

metadata

@@ -0,0 +1,114 @@
+--- !ruby/object:Gem::Specification
+name: opener-pos-tagger-en-es
+version: !ruby/object:Gem::Version
+  version: 2.0.0
+platform: ruby
+authors:
+- development@olery.com
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-05-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: opener-build-tools
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: cucumber
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: POS tagging for english, spanish, dutch, italian and french
+email: 
+executables:
+- pos-tagger-nl
+- pos-tagger-en
+- pos-tagger-fr
+- pos-tagger-es
+- pos-tagger-it
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- bin/pos-tagger-en
+- bin/pos-tagger-es
+- bin/pos-tagger-fr
+- bin/pos-tagger-it
+- bin/pos-tagger-nl
+- core/target/ehu-pos-1.0.jar
+- lib/opener/pos_taggers/en.rb
+- lib/opener/pos_taggers/en/version.rb
+- opener-pos-tagger-en-es.gemspec
+homepage: http://opener-project.github.com/
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 1.9.2
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: POS tagging for english, spanish, dutch, italian and french
+test_files: []
+has_rdoc: yard