opener-pos-tagger-base 2.0.0

data/core/site-packages/pre_build/VUKafParserPy/__init__.py

@@ -0,0 +1,7 @@
+## version = 0.2
+## Added timestamp to function addLinguisitcProcessor
+## 24-april-2013 --> getSingleEntieies and getSingleProperties reads both entities/props in format
+## entities -> entity -> span -> target and entities -> entity -> references -> span
+####
+
+from KafParserMod import KafParser

data/core/token_matcher.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python
+
+
+#####
+# 4-Mar-2013 : modified order of rules to check first if there is a merge and then if it is an extra token
+#  becuase of this case, where can be both:  [ .. . ]  [ . . . ]       
+  
+  
+def add_match(d,id_new,id_ref):
+  if id_new in d:
+    d[id_new].append(id_ref)
+  else:
+    d[id_new]=[id_ref]
+    
+    
+def token_matcher(l_ref,l_new,super_d):
+  #print l_ref
+  #print l_new
+  if len(l_new)==0:
+    return
+  else:
+    token_ref, id_ref = l_ref[0]
+    token_new, id_new = l_new[0]
+    if token_ref == token_new:
+      #print 'Matching ',l_ref[0],l_new[0]
+      #print 'A',l_ref[0],l_new[0]      
+      add_match(super_d,id_new,id_ref)
+      token_matcher(l_ref[1:],l_new[1:],super_d)
+    else:
+      if token_ref.startswith(token_new)  : ##There was an split
+        #print 'D'
+        aux = (token_ref[len(token_new):],id_ref)
+        l_ref[0]=aux
+        
+        add_match(super_d,id_new,id_ref)
+        token_matcher(l_ref,l_new[1:],super_d)
+      
+      elif token_new.startswith(token_ref)  : ##There was a merge
+        #print 'E'
+        aux = (token_new[len(token_ref):],id_new)
+        l_new[0]=aux
+        add_match(super_d,id_new,id_ref)
+        token_matcher(l_ref[1:],l_new,super_d)
+
+
+      elif len(l_new)>1 and l_new[1][0]==token_ref: ## There is an extra token in l_new
+        #print 'B',l_new[1][0],token_ref
+        token_matcher(l_ref[0:],l_new[1:],super_d)
+      
+        
+      elif len(l_ref)>1 and l_ref[1][0] == token_new: ## There is an extra token in l_ref
+        #print 'C',l_ref[1:],l_new[0:]
+        token_matcher(l_ref[1:],l_new[0:],super_d)
+        
+      
+      else: ## Imposible matching
+        #print 'F'
+        #print 'Impossible match of ',l_new[0],l_ref[0]
+        token_matcher(l_ref[1:],l_new[1:],super_d)
+
+
+if __name__ == '__main__':
+  l1 = []
+  s1 = 'Beatrix Wilhelmina Armgard van Oranje -Nassau (Baarn , 31 januari 1938 ) is sinds 30 april 1980 koningin van het Koninkrijk der Nederlanden'
+  
+  s1 = 'Th is is a very simple example'
+  for n,t in enumerate(s1.split(' ')):
+    l1.append((t,'id'+str(n)))
+
+  l2 = []
+  #s2 = 'Beatrix Wilhelmina Armgard van Oranje -Nassau ( Baarn , 31 januari 1938 ) is sinds 30 april 1980 koningin van het Koninkrijk der Nederlanden'
+  s2 = 'This is a very sim ple example'
+  for n,t in enumerate(s2.split(' ')):
+    l2.append((t,'id'+str(n))) 
+
+  super_d = {}
+  token_matcher(l1,l2,super_d)
+  print l1
+  print l2
+  print super_d

data/ext/hack/support.rb

@@ -0,0 +1,38 @@
+require 'opener/build-tools'
+
+include Opener::BuildTools::Requirements
+include Opener::BuildTools::Python
+include Opener::BuildTools::Files
+
+# Directory where packages will be installed to.
+PYTHON_SITE_PACKAGES = File.expand_path(
+  '../../../core/site-packages',
+  __FILE__
+)
+
+# Directory containing the temporary files.
+TMP_DIRECTORY = File.expand_path('../../../tmp', __FILE__)
+
+# Path to the pip requirements file used to install requirements before
+# packaging the Gem.
+PRE_BUILD_REQUIREMENTS = File.expand_path(
+  '../../../pre_build_requirements.txt',
+  __FILE__
+)
+
+# Path to the pip requirements file used to install requirements upon Gem
+# installation.
+PRE_INSTALL_REQUIREMENTS = File.expand_path(
+  '../../../pre_install_requirements.txt',
+  __FILE__
+)
+
+##
+# Verifies the requirements to install thi Gem.
+#
+def verify_requirements
+  require_executable('python')
+  require_version('python', python_version, '2.6.0')
+  require_executable('pip')
+  require_version('pip', pip_version, '1.3.1')
+end

data/lib/opener/pos_taggers/base.rb

@@ -0,0 +1,90 @@
+require 'open3'
+require_relative 'base/version'
+
+module Opener
+  module POSTaggers
+    ##
+    # The base POS tagger that supports Dutch and German.
+    #
+    # @!attribute [r] args
+    #  @return [Array]
+    # @!attribute [r] options
+    #  @return [Hash]
+    #
+    class Base
+      attr_reader :args, :options
+
+      ##
+      # @param [Hash] options
+      #
+      # @option options [Array] :args The commandline arguments to pass to the
+      #  underlying Python script.
+      #
+      def initialize(options = {})
+        @args          = options.delete(:args) || []
+        @options       = options
+      end
+
+      ##
+      # Builds the command used to execute the kernel.
+      #
+      # @return [String]
+      #
+      def command
+        return "#{adjust_python_path} python -E -OO #{kernel} #{args.join(' ')}"
+      end
+
+      ##
+      # Runs the command and returns the output of STDOUT, STDERR and the
+      # process information.
+      #
+      # @param [String] input The input to tag.
+      # @return [Array]
+      #
+      def run(input)
+        return capture(input)
+      end
+
+      protected
+      ##
+      # @return [String]
+      #
+      def adjust_python_path
+        site_packages =  File.join(core_dir, 'site-packages')
+        "env PYTHONPATH=#{site_packages}:$PYTHONPATH"
+      end
+    
+      ##
+      # capture3 method doesn't work properly with Jruby, so 
+      # this is a workaround
+      #
+      def capture(input)
+        Open3.popen3(*command.split(" ")) {|i, o, e, t|
+          out_reader = Thread.new { o.read }
+          err_reader = Thread.new { e.read }
+          i.write input
+          i.close
+          [out_reader.value, err_reader.value, t.value]
+        }
+      end
+
+      ##
+      # @return [String]
+      #
+      def core_dir
+        return File.expand_path('../../../../core', __FILE__)
+      end
+
+      ##
+      # @return [String]
+      #
+      def kernel
+        return File.join(core_dir, 'pos-tagger_open-nlp.py')
+      end
+    end # Base
+
+    class DE < Base
+    end # DE
+
+  end # POSTaggers
+end # Opener

data/lib/opener/pos_taggers/base/version.rb

@@ -0,0 +1,7 @@
+module Opener
+  module POSTaggers
+    class Base
+      VERSION = '2.0.0'
+    end # Base
+  end # POSTaggers
+end # Opener

data/opener-pos-tagger-base.gemspec

@@ -0,0 +1,29 @@
+require File.expand_path('../lib/opener/pos_taggers/base/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = 'opener-pos-tagger-base'
+  gem.version     = Opener::POSTaggers::Base::VERSION
+  gem.authors     = ['development@olery.com']
+  gem.summary     = 'POS tagger for Dutch and German using OpenNLP'
+  gem.description = gem.summary
+  gem.homepage    = 'http://opener-project.github.com/'
+
+  gem.required_ruby_version = '>= 1.9.2'
+
+  gem.files = Dir.glob([
+    'core/**/*',
+    'ext/**/*',
+    'lib/**/*',
+    '*.gemspec',
+    '*_requirements.txt',
+    'README.md'
+  ]).select { |file| File.file?(file) }
+
+  gem.executables = Dir.glob('bin/*').map { |file| File.basename(file) }
+
+  gem.add_dependency 'opener-build-tools', ['>= 0.2.7']
+  gem.add_dependency 'rake'
+
+  gem.add_development_dependency 'rspec'
+  gem.add_development_dependency 'cucumber'
+end

data/pre_build_requirements.txt

@@ -0,0 +1 @@
+git+ssh://git@github.com/opener-project/VU-kaf-parser.git#egg=VUKafParserPy

metadata

@@ -0,0 +1,132 @@
+--- !ruby/object:Gem::Specification
+name: opener-pos-tagger-base
+version: !ruby/object:Gem::Version
+  version: 2.0.0
+platform: ruby
+authors:
+- development@olery.com
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-05-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: opener-build-tools
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.7
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.7
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: cucumber
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: POS tagger for Dutch and German using OpenNLP
+email: 
+executables:
+- pos-tagger-base
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- bin/pos-tagger-base
+- core/mapping.postag.stss.to.opener.csv
+- core/mapping.postag.wotan.to.opener.csv
+- core/opennlp/bin/opennlp
+- core/opennlp/bin/opennlp.bat
+- core/opennlp/lib/jwnl-1.3.3.jar
+- core/opennlp/lib/opennlp-maxent-3.0.2-incubating.jar
+- core/opennlp/lib/opennlp-tools-1.5.2-incubating.jar
+- core/opennlp/lib/opennlp-uima-1.5.2-incubating.jar
+- core/opennlp/models/de-pos-maxent.bin
+- core/opennlp/models/de-pos-perceptron.bin
+- core/opennlp/models/nl-pos-maxent.bin
+- core/opennlp/models/nl-pos-perceptron.bin
+- core/pos-tagger_open-nlp.py
+- core/site-packages/pre_build/VUKafParserPy-1.0-py2.7.egg-info/PKG-INFO
+- core/site-packages/pre_build/VUKafParserPy-1.0-py2.7.egg-info/SOURCES.txt
+- core/site-packages/pre_build/VUKafParserPy-1.0-py2.7.egg-info/dependency_links.txt
+- core/site-packages/pre_build/VUKafParserPy-1.0-py2.7.egg-info/installed-files.txt
+- core/site-packages/pre_build/VUKafParserPy-1.0-py2.7.egg-info/top_level.txt
+- core/site-packages/pre_build/VUKafParserPy/KafDataObjectsMod.py
+- core/site-packages/pre_build/VUKafParserPy/KafDataObjectsMod.pyc
+- core/site-packages/pre_build/VUKafParserPy/KafParserMod.py
+- core/site-packages/pre_build/VUKafParserPy/KafParserMod.pyc
+- core/site-packages/pre_build/VUKafParserPy/__init__.py
+- core/site-packages/pre_build/VUKafParserPy/__init__.pyc
+- core/token_matcher.py
+- ext/hack/support.rb
+- lib/opener/pos_taggers/base.rb
+- lib/opener/pos_taggers/base/version.rb
+- opener-pos-tagger-base.gemspec
+- pre_build_requirements.txt
+homepage: http://opener-project.github.com/
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 1.9.2
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: POS tagger for Dutch and German using OpenNLP
+test_files: []
+has_rdoc: