open-nlp 0.0.1 → 0.1.0

data/README.md

@@ -1,9 +1,4 @@
-**Warning**
-
-This is an alpha release. Expect things to break and/or change in the near future! Also, keep the following in mind:
-
-- Currently, this gem is only tested on JRuby, but support for MRI through Rjb is coming very soon.
-- Currently, the parser and chunker classes are not working.
+[![Build Status](https://secure.travis-ci.org/louismullie/open-nlp.png)](http://travis-ci.org/louismullie/open-nlp)
 
 **About**
 
@@ -13,9 +8,11 @@ This gem only provides a thin wrapper over the OpenNLP API. If you are looking f
 
 **Installing**
 
+_Note: If you are running on MRI, this gem will use the Ruby-Java Bridge (Rjb), which currently does not support Java 7. Therefore, if you have installed Java 7, you should set your JAVA_HOME to point to your old Java 6 install before installing Rjb; for example, `export "JAVA_HOME=/usr/lib/jvm/java-6-openjdk/"`.
+
 First, install the gem: `gem install open-nlp`. Then, individually download the appropriate models from the [open-nlp website](http://opennlp.sourceforge.net/models-1.5/) or just get [all english language models](louismullie.com/treat/open-nlp-english.zip) in one package (80 MB).
 
-Place the contents of the extracted archive inside the /bin/ folder of the stanford-core-nlp gem (e.g. [...]/gems/open-nlp-0.x.x/bin/).
+Place the contents of the extracted archive inside the /bin/ folder of the open-nlp gem (e.g. [...]/gems/open-nlp-0.x.x/bin/).
 
 **Configuration**
 
@@ -43,7 +40,7 @@ OpenNLP.log_file = 'log.txt'
 # OpenNLP.use(:french) # or :german
 # 
 # Change a specific model file.
-# StanfordCoreNLP.set_model('pos.model', 'english-left3words-distsim.tagger')
+# OpenNLP.set_model('pos.model', 'english-left3words-distsim.tagger')
 ```
 
 **Using the gem**

data/bin/Utils.java

@@ -0,0 +1 @@
+import java.util.Arrays;
    
    public static String[] tagWithArrayList(POSTagger posTagger, ArrayList[] objectArray) {
      return posTagger.tag(getStringArray(objectArray));
    }
    public static Object[] findWithArrayList(NameFinderME nameFinder, ArrayList[] tokens) {
      return nameFinder.find(getStringArray(tokens));
    }
    public static Object[] chunkWithArrays(ChunkerME chunker, ArrayList[] tokens, ArrayList[] tags) {
      return chunker.chunk(getStringArray(tokens), getStringArray(tags));
    }
    public static String[] getStringArray(ArrayList[] objectArray) {
      String[] stringArray = Arrays.copyOf(objectArray, objectArray.length, String[].class);
    }

data/lib/open-nlp.rb

@@ -1,162 +1,31 @@
 module OpenNLP
-  
-  # Library version.
-   VERSION = '0.0.1'
 
-   # Require configuration.
-   require 'open-nlp/config'
-  
-  # ############################ #
-  # BindIt Configuration Options #
-  # ############################ #
-  
-   require 'bind-it'
-   extend BindIt::Binding
-
-  # The path in which to look for JAR files, with
-  # a trailing slash (default is gem's bin folder).
-  self.jar_path = File.dirname(__FILE__) + '/../bin/'
-  
-  # Load the JVM with a minimum heap size of 512MB,
-  # and a maximum heap size of 1024MB.
-  self.jvm_args = ['-Xms512M', '-Xmx1024M']
-  
-  # Turn logging off by default.
-  self.log_file = nil
-  
-  # Default JARs to load.
-  self.default_jars = [
-    'jwnl-1.3.3.jar',
-    'opennlp-tools-1.5.2-incubating.jar',
-    'opennlp-maxent-3.0.2-incubating.jar',
-    'opennlp-uima-1.5.2-incubating.jar'
-  ]
-  
-  # Default namespace.
-  self.default_namespace = 'opennlp.tools'
-  
-  # Default classes.
-  self.default_classes = [
-    ['DocumentCategorizerME', 'opennlp.tools.doccat'],
-    ['ChunkerME', 'opennlp.tools.chunker'],
-    ['DictionaryDetokenizer', 'opennlp.tools.tokenize'],
-    ['NameFinderME', 'opennlp.tools.namefind'],
-    ['Parse', 'opennlp.tools.parser'],
-    ['ParserFactory', 'opennlp.tools.parser'],
-    ['POSTaggerME', 'opennlp.tools.postag'],
-    ['SentenceDetectorME', 'opennlp.tools.sentdetect'],
-    ['SimpleTokenizer', 'opennlp.tools.tokenize'],
-    ['TokenizerME', 'opennlp.tools.tokenize']
-  ]
-  
-  # Redefine the Bind-It class loader to redefine 
-  # a new constructor for classes that require a model.
-  def self.load_klass(klass, base, name = nil)
-    super(klass,base,name)
-    requires_model = OpenNLP::Config::RequiresModel
-    return unless requires_model.include?(klass)
-    new_class = Class.new(const_get(klass)) do
-      def initialize(file = nil, *args)
-        klass = OpenNLP.last_name(self.class)
-        if !file && !OpenNLP.has_default_model?(klass)
-          raise 'This class intentionally has no default ' +
-          'model. Please supply a file name as an argument ' +
-          'to the class constructor.'
-        else
-          model = OpenNLP.get_model(klass, file)
-          super(*([model] + args))
-        end
-      end
-    end
-    remove_const(klass)
-    const_set(klass, new_class)
-  end
-  
-  # Make the bindings.
-  self.bind
-  
-  # Load utility classes.
-  self.load_class('FileInputStream', 'java.io')
-  
-  # ############################ #
-  #   OpenNLP bindings proper    #
-  # ############################ #
+  # Library version.
+  VERSION = '0.1.0'
 
-  class <<self
-    # A hash containing loaded models.
-    attr_accessor :models
-    # A hash containing the names of loaded models.
-    attr_accessor :model_files
-    # The folder in which to look for models.
-    attr_accessor :model_path
-    # Store the language currently being used.
-    attr_accessor :language
-  end
-  
-  # The loaded models.
-  self.models = {}
-  
-  # The names of loaded models.
-  self.model_files = {}
-  
-  # The path to the main folder containing the folders
-  # with the individual models inside. By default, this
-  # is the same as the JAR path.
-  self.model_path = self.jar_path
+  # Require Java bindings.
+  require 'open-nlp/bindings'
+  OpenNLP::Bindings.bind
   
-  # Default the language to English.
-  self.language = :english
+  # Require Ruby wrappers.
+  require 'open-nlp/classes'
   
-  # Use a given language for default models.
-  def self.use(language)
-    self.language = language
+  # Load a Java class into the OpenNLP
+  # namespace (e.g. OpenNLP::Loaded).
+  def load_class(*args)
+    OpenNLP::Bindings.load_class(*args)
   end
   
-  def self.set_model
-    # Implement
-  end
-
-  def self.has_default_model?(klass)
-    name = OpenNLP::Config::ClassToName[klass]
-    if !OpenNLP::Config::DefaultModels[name]
-      raise 'No default model files are available ' +
-      "for the class #{klass}. Please supply a model " +
-      'as an argument to the constructor.'
-    end
-    !OpenNLP::Config::DefaultModels[name].empty?
+  # Forwards the handling of missing
+  # constants to the Bindings class.
+  def const_missing(const)
+    OpenNLP::Bindings.const_get(const)
   end
   
-  def self.get_model(klass, file=nil)
-    name = OpenNLP::Config::ClassToName[klass]
-    if !self.language and !file
-      raise 'No model file was supplied to the ' +
-      'constructor. Please supply a model file ' +
-      'or call OpenNLP.use(:some_language), to ' +
-      'load the default models for a language.'
-    end
-    OpenNLP.load_model(name, file)
-    model = OpenNLP.models[name]
-  end
-
-  def self.load_model(name, file = nil)
-    if self.models[name] && file ==
-       self.model_files[name]
-       return self.models[name]
-    end
-    models = Config::DefaultModels[name]
-    file ||= models[self.language]
-    path = self.model_path + file
-    stream = FileInputStream.new(path)
-    klass = Config::NameToClass[name]
-    load_class(*klass)
-    klass = const_get(klass[0])
-    model = klass.new(stream)
-    self.model_files[name] = file
-    self.models[name] = model
+  # Forward the handling of missing 
+  # methods to the Bindings class.
+  def method_missing(sym, *args, &block)
+    OpenNLP::Bindings.send(sym, *args, &block)
   end
 
-  def self.last_name(klass)
-    klass.to_s.split('::')[-1]
-  end
-  
-end
+end

data/lib/open-nlp/base.rb

@@ -0,0 +1,49 @@
+class OpenNLP::Base
+
+  def initialize(file_or_arg=nil, *args)
+
+    @proxy_class = OpenNLP::Bindings.const_get(last_name)
+
+    if requires_model?
+      if !file_or_arg && !has_default_model?
+        raise "No default model files are available for " +
+        "class #{last_name}. Please supply a model as" +
+        "an argument to the constructor."
+      end
+      @model = OpenNLP::Bindings.get_model(last_name, file_or_arg)
+      @proxy_inst = @proxy_class.new(*([@model] + args))
+    else
+      @proxy_inst = @proxy_class.new(*([*file_or_arg] + args))
+    end
+
+  end
+
+  def has_default_model?
+    name = OpenNLP::Config::ClassToName[last_name]
+    !OpenNLP::Config::DefaultModels[name].empty?
+  end
+
+  def requires_model?
+    OpenNLP::Config::RequiresModel.include?(last_name)
+  end
+
+  def last_name
+    self.class.to_s.split('::')[-1]
+  end
+
+
+  def method_missing(sym, *args, &block)
+    @proxy_inst.send(sym, *args, &block)
+  end
+  
+  protected
+
+  def get_list(tokens)
+    list = OpenNLP::Bindings::ArrayList.new
+    tokens.each do |t|
+      list.add(OpenNLP::Bindings::String.new(t.to_s))
+    end
+    list
+  end
+
+end

data/lib/open-nlp/bindings.rb

@@ -0,0 +1,133 @@
+module OpenNLP::Bindings
+
+  # Require configuration.
+  require 'open-nlp/config'
+
+  # ############################ #
+  # BindIt Configuration Options #
+  # ############################ #
+
+  require 'bind-it'
+  extend BindIt::Binding
+
+  # The path in which to look for JAR files, with
+  # a trailing slash (default is gem's bin folder).
+  self.jar_path = File.dirname(__FILE__) + '/../../bin/'
+
+  # Load the JVM with a minimum heap size of 512MB,
+  # and a maximum heap size of 1024MB.
+  self.jvm_args = ['-Xms512M', '-Xmx1024M']
+
+  # Turn logging off by default.
+  self.log_file = nil
+
+  # Default JARs to load.
+  self.default_jars = [
+    'jwnl-1.3.3.jar',
+    'opennlp-tools-1.5.2-incubating.jar',
+    'opennlp-maxent-3.0.2-incubating.jar',
+    'opennlp-uima-1.5.2-incubating.jar'
+  ]
+
+  # Default namespace.
+  self.default_namespace = 'opennlp.tools'
+
+  # Default classes.
+  self.default_classes = [
+    ['AbstractBottomUpParser', 'opennlp.tools.parser'],
+    ['DocumentCategorizerME', 'opennlp.tools.doccat'],
+    ['ChunkerME', 'opennlp.tools.chunker'],
+    ['DictionaryDetokenizer', 'opennlp.tools.tokenize'],
+    ['NameFinderME', 'opennlp.tools.namefind'],
+    ['Parser', 'opennlp.tools.parser.chunking'],
+    ['Parse', 'opennlp.tools.parser'],
+    ['ParserFactory', 'opennlp.tools.parser'],
+    ['POSTaggerME', 'opennlp.tools.postag'],
+    ['SentenceDetectorME', 'opennlp.tools.sentdetect'],
+    ['SimpleTokenizer', 'opennlp.tools.tokenize'],
+    ['Span', 'opennlp.tools.util'],
+    ['TokenizerME', 'opennlp.tools.tokenize']
+  ]
+  
+  # Add in Rjb workarounds.
+  unless RUBY_PLATFORM =~ /java/
+    self.default_jars << 'utils.jar'
+    self.default_classes << ['Utils', '']
+  end
+  
+  # Make the bindings.
+  self.bind
+
+  # Load utility classes.
+  self.load_class('FileInputStream', 'java.io')
+  self.load_class('String', 'java.lang')
+  self.load_class('ArrayList', 'java.util')
+
+  # ############################ #
+  #   OpenNLP bindings proper    #
+  # ############################ #
+
+  class <<self
+    # A hash containing loaded models.
+    attr_accessor :models
+    # A hash containing the names of loaded models.
+    attr_accessor :model_files
+    # The folder in which to look for models.
+    attr_accessor :model_path
+    # Store the language currently being used.
+    attr_accessor :language
+  end
+
+  # The loaded models.
+  self.models = {}
+
+  # The names of loaded models.
+  self.model_files = {}
+
+  # The path to the main folder containing the folders
+  # with the individual models inside. By default, this
+  # is the same as the JAR path.
+  self.model_path = self.jar_path
+
+  # Default the language to English.
+  self.language = :english
+
+  # Use a given language for default models.
+  def self.use(language)
+    self.language = language
+  end
+
+  def self.get_model(klass, file=nil)
+    name = OpenNLP::Config::ClassToName[klass]
+    if !self.language and !file
+      raise 'No model file was supplied to the ' +
+      'constructor. Please supply a model file ' +
+      'or call OpenNLP.use(:some_language), to ' +
+      'load the default models for a language.'
+    end
+    self.load_model(name, file)
+    model = self.models[name]
+  end
+
+  def self.set_model
+    raise 'Not implemented.'
+  end
+
+  def self.load_model(name, file = nil)
+    if self.models[name] && file ==
+      self.model_files[name]
+      return self.models[name]
+    end
+    models = OpenNLP::Config::DefaultModels[name]
+    file ||= models[self.language]
+    path = self.model_path + file
+    stream = FileInputStream.new(path)
+    klass = OpenNLP::Config::NameToClass[name]
+    load_class(*klass) unless const_defined?(klass[0])
+    klass = const_get(klass[0])
+    model = klass.new(stream)
+    self.model_files[name] = file
+    self.models[name] = model
+  end
+
+end

data/lib/open-nlp/classes.rb

@@ -0,0 +1,77 @@
+require 'open-nlp/base'
+
+class OpenNLP::SentenceDetectorME < OpenNLP::Base; end
+
+class OpenNLP::SimpleTokenizer < OpenNLP::Base; end
+
+class OpenNLP::TokenizerME < OpenNLP::Base; end
+
+class OpenNLP::POSTaggerME < OpenNLP::Base
+    
+  unless RUBY_PLATFORM =~ /java/
+    def tag(*args)
+      OpenNLP::Bindings::Utils.tagWithArrayList(@proxy_inst, args[0])
+    end
+  end
+  
+end
+
+class OpenNLP::ChunkerME < OpenNLP::Base
+
+  if RUBY_PLATFORM =~ /java/
+
+    def chunk(tokens, tags)
+      if !tokens.is_a?(Array)
+        tokens = tokens.to_a
+        tags = tags.to_a
+      end
+      tokens = tokens.to_java(:String)
+      tags = tags.to_java(:String)
+      @proxy_inst.chunk(tokens,tags).to_a
+    end
+
+  else
+
+    def chunk(tokens, tags)
+      chunks = OpenNLP::Bindings::Utils.chunkWithArrays(@proxy_inst, tokens,tags)
+      chunks.map { |c| c.to_s }
+    end
+
+  end
+
+end
+
+class OpenNLP::Parser < OpenNLP::Base
+
+  def parse(text)
+
+    tokenizer = OpenNLP::TokenizerME.new
+    full_span = OpenNLP::Bindings::Span.new(0, text.size)
+
+    parse_obj = OpenNLP::Bindings::Parse.new(
+    text, full_span, "INC", 1, 0)
+
+    tokens = tokenizer.tokenize_pos(text)
+
+    tokens.each_with_index do |tok,i|
+      start, stop = tok.get_start, tok.get_end
+      token = text[start..stop-1]
+      span = OpenNLP::Bindings::Span.new(start, stop)
+      parse = OpenNLP::Bindings::Parse.new(text, span, "TK", 0, i)
+      parse_obj.insert(parse)
+    end
+
+    @proxy_inst.parse(parse_obj)
+
+  end
+
+end
+
+class OpenNLP::NameFinderME < OpenNLP::Base
+  unless RUBY_PLATFORM =~ /java/
+    def find(*args)
+      OpenNLP::Bindings::Utils
+      .findWithArrayList(@proxy_inst, args[0])
+    end
+  end
+end

data/lib/open-nlp/config.rb

@@ -17,7 +17,7 @@ module OpenNLP::Config
     'DocumentCategorizerME' => :categorizer,
     'NameFinderME' => :name_finder,
     'POSTaggerME' => :pos_tagger,
-    'ParserME' => :parser,
+    'Parser' => :parser,
     'SentenceDetectorME' => :sentence_detector,
     'TokenizerME' => :tokenizer,
   }
@@ -48,7 +48,7 @@ module OpenNLP::Config
   # Classes that require a model as first argument to constructor.
   RequiresModel = [
     'SentenceDetectorME', 'NameFinderME', 'DictionaryDetokenizer',
-    'TokenizerME', 'ChunkerME', 'POSTaggerME'
+    'TokenizerME', 'ChunkerME', 'POSTaggerME', 'Parser'
   ]
 
   

data/spec/english_spec.rb

@@ -3,7 +3,6 @@ require_relative 'spec_helper'
 
 describe OpenNLP do
 
-  # Failing spec #1
   context "the maximum entropy chunker is run after tokenization and POS tagging" do
     it "should find the accurate chunks" do
       
@@ -15,28 +14,38 @@ describe OpenNLP do
       tokens = tokenizer.tokenize(sent)
       tags   = tagger.tag(tokens)
       
-      chunks = chunker.chunk(tokens.to_java(:String), pos_tags.to_java(:String))
-      # cannot convert instance of class org.jruby.java.proxies.ArrayJavaProxy to class java.lang.String
-      
+      chunks = chunker.chunk(tokens, tags)
+
+      chunks.to_a.should eql %w[B-NP I-NP B-PP B-NP I-NP B-VP I-VP B-PP B-NP I-NP O]
       tokens.to_a.should eql %w[The death of the poet was kept from his poems .]
-      tags.should eql ['put tags here']
+      tags.to_a.should eql %w[DT NN IN DT NN VBD VBN IN PRP$ NNS .]
       
     end
   end
 
-  # Failing spec #2
   context "the maximum entropy parser is run after tokenization" do
     it "parses the text accurately" do
+
       sent      = "The death of the poet was kept from his poems."
-      tokenizer = OpenNLP::TokenizerME.new
-      p_model   = OpenNLP.load_model(:parser)
-      parser    = OpenNLP::ParserFactory.create(p_model)
-      tokens = tokenizer.tokenize(sent)
-      result = parser.parse(tokens.to_java(:String))
-      # cannot convert instance of class org.jruby.java.proxies.ArrayJavaProxy to class java.lang.String
-      # org/jruby/java/addons/KernelJavaAddons.java:70:in `to_java'
-      # /ruby/gems/open-nlp/spec/english_spec.rb in `(root)'
-      puts result.to_a.inspect
+      parser = OpenNLP::Parser.new
+      parse = parser.parse(sent)
+
+      parse.get_text.should eql sent
+
+      parse.get_span.get_start.should eql 0
+      parse.get_span.get_end.should eql 46
+      parse.get_span.get_type.should eql nil # ?
+      parse.get_child_count.should eql 1
+
+      child = parse.get_children[0]
+
+      child.text.should eql "The death of the poet was kept from his poems."
+      child.get_child_count.should eql 3
+      child.get_head_index.should eql 5
+
+      child.get_head.get_child_count.should eql 1
+      child.get_type.should eql "S"
+
     end
   end
 

metadata

@@ -1,32 +1,48 @@
 --- !ruby/object:Gem::Specification
 name: open-nlp
 version: !ruby/object:Gem::Version
-  prerelease:
-  version: 0.0.1
+  version: 0.1.0
+  prerelease: 
 platform: ruby
 authors:
 - Louis Mullie
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-19 00:00:00.000000000 Z
+date: 2012-12-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bind-it
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
     none: false
-  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
     none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
   prerelease: false
-  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: ! ' Ruby bindings to the OpenNLP tools, a Java machine learning toolkit
   for natural language processing (NLP). '
 email:
@@ -39,8 +55,13 @@ files:
 - bin/opennlp-maxent-3.0.2-incubating.jar
 - bin/opennlp-tools-1.5.2-incubating.jar
 - bin/opennlp-uima-1.5.2-incubating.jar
-- lib/open-nlp.rb
+- bin/utils.jar
+- bin/Utils.java
+- lib/open-nlp/base.rb
+- lib/open-nlp/bindings.rb
+- lib/open-nlp/classes.rb
 - lib/open-nlp/config.rb
+- lib/open-nlp.rb
 - spec/english_spec.rb
 - spec/sample.txt
 - spec/spec_helper.rb
@@ -48,27 +69,26 @@ files:
 - LICENSE
 homepage: https://github.com/louismullie/open-nlp
 licenses: []
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
-  none: false
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
-  none: false
 requirements: []
-rubyforge_project:
+rubyforge_project: 
 rubygems_version: 1.8.24
-signing_key:
+signing_key: 
 specification_version: 3
 summary: Ruby bindings to the OpenNLP Java toolkit.
 test_files: []
-...