opener-ner-base 3.0.1 → 3.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8981a1d8d5e60e579ee5fd0e0e8ce4f6b0dbf6c9
-  data.tar.gz: 56faadbc7debd07516a5f72c0403c2496b5a7b42
+  metadata.gz: 184b5a02199988f3fa5e798e9438b679966542e0
+  data.tar.gz: e60c01830e0f3e97b1a7489cb4c5acfdc9d3cd14
 SHA512:
-  metadata.gz: f943839a5f64c77d71555ca096707481d93242473eb7663cc96a3904991b653a062eb1a7f62cda783e89b2f2824c32d839f95d1a61949f45b4cf9d021466ebf3
-  data.tar.gz: b2799464a0a2da1af3af91aa8282f5a53925e4dab639becb0deeb6cfb122b5c985d19168da8f08b8622989dcf6c55990eccdb98644934d0fa9213b39eca01c77
+  metadata.gz: ca7ed76f8a08c1409ed6de52a2178e766994816ebf92e89821700b0f7cddf0b1e6c0f59e0c69009fbd9877149c84d00dc60efa7ab24be22b403ed4bbb3bbdaec
+  data.tar.gz: 7852dd522d262eb94f9482f4a0a81422ba66bd9b762d52a5bf54b79e3192161cd0039b53a2f70f075ece4a93bfb2d59dfd6c763235be188858dbe95dd9141f47

data/README.md

@@ -1,5 +1,3 @@
-[![Build Status](https://drone.io/github.com/opener-project/ner-base/status.png)](https://drone.io/github.com/opener-project/ner-base/latest)
-
 # NER Base
 
 This repository contains the source code used for performing Named Entity
@@ -42,7 +40,14 @@ Using specific install:
 
 ## Usage
 
-    cat some_input_file.kaf | ner-en
+Basic usage:
+
+    cat some_input_file.kaf | ner-base
+
+This component ships a built-in set of models. If you have your own models you
+can set the environment variable `NER_BASE_MODELS_PATH` to the directory
+containing your models. Each model should be named `LANGUAGE.bin` where
+`LANGUAGE` is a 2 letter language code (`nl`. `en`, etc).
 
 ## Contributing
 

data/lib/opener/ners/base.rb

@@ -1,74 +1,36 @@
-require 'open3'
 require 'stringio'
-require 'nokogiri'
-
-require File.expand_path("../../../../core/target/ixa-pipe-nerc-1.1.0.jar", __FILE__)
+require 'oga'
+require 'opener/core'
 
 require_relative 'base/version'
 
+require File.expand_path("../../../../core/target/ixa-pipe-nerc-1.5.2.jar", __FILE__)
+
 module Opener
   module Ners
     ##
     # Base NER class that supports various languages such as Dutch and English.
     #
-    # @!attribute [r] options
-    #  @return [Hash]
-    #
-    # @!attribute [r] features
-    #  @return [String]
-    #
-    # @!attribute [r] beamsize
-    #  @return [Fixnum]
-    #
-    # @!attribute [r] dictionaries
-    #  @return [String]
-    #
-    # @!attribute [r] dictionaries_path
-    #  @return [String]
-    #
-    # @!attribute [r] lexer
-    #  @return [Fixnum]
-    #
-    # @!attribute [r] model
-    #  @return [String]
-    #
-    # @!attribute [r] enable_time
-    #  @return [TrueClass|FalseClass]
-    #
     class Base
-      attr_reader :features, :beamsize, :dictionaries, :dictionaries_path,
-        :lexer, :model, :enable_time
+      # The default models directory.
+      MODELS_PATH = File.expand_path('../../../../models', __FILE__)
+
+      # @return [String]
+      attr_reader :models
+
+      # @return [TrueClass|FalseClass]
+      attr_reader :enable_time
 
       ##
       # @param [Hash] options
       #
-      # @option options [String] :features The NERC feature to use, defaults to
-      #  "baseline".
-      #
-      # @option options [Fixnum] :beamsize The beam size for decoding, defaults
-      #  to 3.
-      #
-      # @option options [String] :dictionaries The dictionary to use, if any.
-      #
-      # @option options [String] :dictionaries_path The path to the
-      #  dictionaries.
-      #
-      # @option options [Fixnum] :lexer The lexer rules to use for NERC
-      #  tagging.
-      #
-      # @option options [String] :model The model to use for NERC annotation.
-      #
       # @option options [TrueClass|FalseClass] :enable_time Whether or not to
       #  enable dynamic timestamps (enabled by default).
       #
       def initialize(options = {})
-        @dictionaries      = options[:dictionaries]
-        @dictionaries_path = options[:dictionaries_path]
-        @features          = options.fetch(:features, 'baseline')
-        @beamsize          = options.fetch(:beamsize, 3)
-        @lexer             = options[:lexer]
-        @model             = options.fetch(:model, 'default')
-        @enable_time       = options.fetch(:enable_time, true)
+        @models = ENV['NER_BASE_MODELS_PATH'] || MODELS_PATH
+
+        @enable_time = options.fetch(:enable_time, true)
       end
 
       ##
@@ -79,19 +41,16 @@ module Opener
       # @return [Array]
       #
       def run(input)
-        lang = language_from_kaf(input)
-        kaf  = new_kaf_document(input)
-        args = [lang, model, features, beamsize]
+        lang  = language_from_kaf(input)
+        model = File.join(models, "#{lang}.bin")
 
-        if use_dictionaries?
-          args += [dictionaries, dictionaries_path, lexer]
-        end
+        raise(Core::UnsupportedLanguageError, lang) unless File.file?(model)
 
-        annotator = Java::es.ehu.si.ixa.pipe.nerc.Annotate.new(*args)
+        kaf        = new_kaf_document(input)
+        properties = build_properties(lang, model)
+        annotator  = Java::eus.ixa.ixa.pipe.nerc.Annotate.new(properties)
 
         annotator.annotate_kaf(enable_time, kaf)
-
-        return kaf.to_string
       end
 
       ##
@@ -102,14 +61,7 @@ module Opener
         input_io = StringIO.new(input)
         reader   = Java::java.io.InputStreamReader.new(input_io.to_inputstream)
 
-        return Java::ixa.kaflib.KAFDocument.create_from_stream(reader)
-      end
-
-      ##
-      # @return [TrueClass|FalseClass]
-      #
-      def use_dictionaries?
-        return dictionaries || dictionaries_path || features == 'dict'
+        Java::ixa.kaflib.KAFDocument.create_from_stream(reader)
       end
 
       ##
@@ -119,9 +71,39 @@ module Opener
       # @return [String]
       #
       def language_from_kaf(input)
-        document = Nokogiri::XML(input)
+        parser   = Oga::XML::PullParser.new(input)
+        language = nil
+
+        parser.parse do |node|
+          if node.is_a?(Oga::XML::Element) and node.name == 'KAF'
+            language = node.get('xml:lang')
+            break
+          end
+        end
+
+        # Make sure nobody can _somehow_ inject a language such as "../../foo".
+        unless language =~ /\A[a-zA-Z\-_]+\z/
+          raise Core::UnsupportedLanguageError, language
+        end
+
+        language
+      end
+
+      private
+
+      # @param [String] language
+      # @param [String] model
+      def build_properties(language, model)
+        properties = Java::java.util.Properties.new
+
+        properties.set_property('language', language)
+        properties.set_property('model', model)
+        properties.set_property('ruleBasedOption', 'off')
+        properties.set_property('dictTag', 'off')
+        properties.set_property('dictPath', 'off')
+        properties.set_property('clearFeatures', 'no')
 
-        return document.at('KAF').attr('xml:lang')
+        properties
       end
     end # Base
   end # Ners

data/lib/opener/ners/base/version.rb

@@ -1,7 +1,7 @@
 module Opener
   module Ners
     class Base
-      VERSION = '3.0.1'
+      VERSION = '3.1.0'
     end # Base
   end # Ners
 end # Opener

data/opener-ner-base.gemspec

@@ -12,6 +12,7 @@ Gem::Specification.new do |gem|
   gem.files = Dir.glob([
     'core/target/ixa-pipe-nerc-*.jar',
     'lib/**/*',
+    'models/**/*',
     '*.gemspec',
     'README.md',
     'LICENSE.txt'
@@ -19,7 +20,8 @@ Gem::Specification.new do |gem|
 
   gem.executables = Dir.glob('bin/*').map { |file| File.basename(file) }
 
-  gem.add_dependency 'nokogiri'
+  gem.add_dependency 'oga'
+  gem.add_dependency 'opener-core'
 
   gem.add_development_dependency 'rspec', '~> 3.0'
   gem.add_development_dependency 'cucumber'

metadata

@@ -1,85 +1,99 @@
 --- !ruby/object:Gem::Specification
 name: opener-ner-base
 version: !ruby/object:Gem::Version
-  version: 3.0.1
+  version: 3.1.0
 platform: ruby
 authors:
 - development@olery.com
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2014-08-28 00:00:00.000000000 Z
+date: 2015-09-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: oga
+  prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: opener-core
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: rspec
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+  name: rspec
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: cucumber
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: cucumber
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: rake
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: rake
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: cliver
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: cliver
   prerelease: false
   type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Base NER component for languages such as English.
 email:
 executables:
@@ -90,9 +104,15 @@ files:
 - LICENSE.txt
 - README.md
 - bin/ner-base
-- core/target/ixa-pipe-nerc-1.1.0.jar
+- core/target/ixa-pipe-nerc-1.5.2.jar
 - lib/opener/ners/base.rb
 - lib/opener/ners/base/version.rb
+- models/de.bin
+- models/en.bin
+- models/es.bin
+- models/fr.bin
+- models/it.bin
+- models/nl.bin
 - opener-ner-base.gemspec
 homepage: http://opener-project.github.com/
 licenses:
@@ -104,18 +124,19 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.2.2
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: Base NER component for languages such as English.
 test_files: []
+has_rdoc: