open_nlp 0.0.7-java → 0.1.0-java

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 57b451e151cf3a0ed7d28b9451f48b7ec7a31288
+  data.tar.gz: fc5e0b61f0baf6673a4a3dbc05e4c7828d2a935c
+SHA512:
+  metadata.gz: 420670f2e006071f47ba5d18f6152427ba49745b2647a7b5e7520acc987575ce02cc0e3cecbd3417f65d7d4c30dfe63740ac9c422a1f43d9a0ed5bb1b6a572a1
+  data.tar.gz: 059bf4f29726f027b634d802cb8f4ff1b1cb9b04ef6315a4d5e77d44098e1eed1a651edd97cc7d37a6ed4b91a6a6e03bb571aef45e996919fcfbec481ce5f269

data/.gitignore

@@ -3,7 +3,6 @@
 .bundle
 .config
 .yardoc
-Gemfile.lock
 InstalledFiles
 _yardoc
 coverage
@@ -15,4 +14,5 @@ spec/reports
 test/tmp
 test/version_tmp
 tmp
-.idea
+.idea
+.gems

data/.ruby-version

@@ -0,0 +1 @@
+jruby-9.0.5.0

data/.travis.yml

@@ -0,0 +1,6 @@
+language: ruby
+rvm:
+  - jruby-19mode
+  - jruby-1.7.20
+  - jruby-9.0.5.0
+script: JRUBY_OPTS=-J-Xmx768m bundle exec rspec spec

data/Gemfile.lock

@@ -0,0 +1,31 @@
+PATH
+  remote: .
+  specs:
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    diff-lcs (1.2.5)
+    rspec (3.4.0)
+      rspec-core (~> 3.4.0)
+      rspec-expectations (~> 3.4.0)
+      rspec-mocks (~> 3.4.0)
+    rspec-core (3.4.4)
+      rspec-support (~> 3.4.0)
+    rspec-expectations (3.4.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.4.0)
+    rspec-mocks (3.4.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.4.0)
+    rspec-support (3.4.1)
+
+PLATFORMS
+  java
+
+DEPENDENCIES
+  open_nlp!
+  rspec
+
+BUNDLED WITH
+   1.11.2

data/README.md

@@ -1,5 +1,7 @@
 # OpenNlp
 
+[![Build Status](https://travis-ci.org/hck/open_nlp.png?branch=master)](https://travis-ci.org/hck/open_nlp) [![Code Climate](https://codeclimate.com/github/hck/open_nlp.png)](https://codeclimate.com/github/hck/open_nlp)
+
 A JRuby wrapper for the Apache OpenNLP tools library, that allows you execute common natural language processing tasks, such as
  * sentence detection
  * tokenize
@@ -35,8 +37,13 @@ Then you can create instances of open_nlp classes and use it for your nlp tasks
 
     sentence_detect_model = OpenNlp::Model::SentenceDetector.new("nlp_models/en-sent.bin")
     sentence_detector = OpenNlp::SentenceDetector.new(sentence_detect_model)
+
+    # get sentences as array of strings
     sentence_detector.detect('The red fox sleeps soundly.')
 
+    # get array of OpenNLP::Util::Span objects:
+    sentence_detector.pos_detect('"The sky is blue. The Grass is green."')
+
 ### Tokenize
 
     token_model = OpenNlp::Model::Tokenizer.new("nlp_models/en-token.bin")
@@ -92,4 +99,4 @@ Then you can create instances of open_nlp classes and use it for your nlp tasks
 2. Create your feature branch (`git checkout -b my-new-feature`)
 3. Commit your changes (`git commit -am 'Add some feature'`)
 4. Push to the branch (`git push origin my-new-feature`)
-5. Create new Pull Request
+5. Create new Pull Request

data/lib/open_nlp.rb

@@ -1,8 +1,8 @@
 require 'open_nlp/version'
 
 require 'java'
-require 'open_nlp/opennlp-tools-1.5.2-incubating.jar'
-require 'open_nlp/opennlp-maxent-3.0.2-incubating.jar'
+require 'open_nlp/opennlp-tools-1.5.3.jar'
+require 'open_nlp/opennlp-maxent-3.0.3.jar'
 
 require 'open_nlp/java_class'
 
@@ -28,4 +28,4 @@ require 'open_nlp/sentence_detector'
 require 'open_nlp/tokenizer'
 require 'open_nlp/chunker'
 require 'open_nlp/parser'
-require 'open_nlp/parser/parse'
+require 'open_nlp/parser/parse'

data/lib/open_nlp/categorizer.rb

@@ -2,11 +2,15 @@ module OpenNlp
   class Categorizer < Tool
     self.java_class = Java::opennlp.tools.doccat.DocumentCategorizerME
 
+    # Categorizes a string passed as parameter to one of the categories
+    #
+    # @param [String] str string to be categorized
+    # @return [String] category
     def categorize(str)
-      raise ArgumentError, "str must be a String" unless str.is_a?(String)
+      fail ArgumentError, 'str param must be a String' unless str.is_a?(String)
 
-      outcomes = @j_instance.categorize(str)
-      @j_instance.getBestCategory(outcomes)
+      outcomes = j_instance.categorize(str)
+      j_instance.getBestCategory(outcomes)
     end
   end
 end

data/lib/open_nlp/chunker.rb

@@ -5,27 +5,38 @@ module OpenNlp
     def initialize(model, token_model, pos_model)
       super(model)
 
-      raise ArgumentError, "model must be an OpenNlp::Tokenizer::Model" unless token_model.is_a?(Model::Tokenizer)
-      raise ArgumentError, "model must be an OpenNlp::POSTagger::Model" unless pos_model.is_a?(Model::POSTagger)
+      unless token_model.is_a?(Model::Tokenizer)
+        fail ArgumentError, 'token model must be an OpenNlp::Tokenizer::Model'
+      end
+
+      unless pos_model.is_a?(Model::POSTagger)
+        fail ArgumentError, 'pos model must be an OpenNlp::POSTagger::Model'
+      end
 
       @tokenizer = Tokenizer.new(token_model)
       @pos_tagger = POSTagger.new(pos_model)
     end
 
+    # Chunks a string into part-of-sentence pieces
+    #
+    # @param [String] str string to chunk
+    # @return [Array] array of chunks with part-of-sentence information
     def chunk(str)
-      raise ArgumentError, "str must be a String" unless str.is_a?(String)
+      fail ArgumentError, 'str must be a String' unless str.is_a?(String)
 
-      tokens = @tokenizer.tokenize(str)
-      pos_tags = @pos_tagger.tag(tokens).to_ary
+      tokens = tokenizer.tokenize(str)
+      pos_tags = pos_tagger.tag(tokens).to_ary
 
-      chunks = @j_instance.chunk(tokens.to_java(:String), pos_tags.to_java(:String)).to_ary
+      chunks = j_instance.chunk(tokens.to_java(:String), pos_tags.to_java(:String)).to_ary
 
       build_chunks(chunks, tokens, pos_tags)
     end
 
     private
+
+    attr_reader :tokenizer, :pos_tagger
+
     def build_chunks(chunks, tokens, pos_tags)
-      # data[i] = [token, pos_tag, chunk_val]
       data = tokens.zip(pos_tags, chunks)
 
       data.inject([]) do |acc, val|
@@ -45,7 +56,7 @@ module OpenNlp
     end
 
     def get_last_probabilities
-      @j_instance.probs.to_ary
+      j_instance.probs.to_ary
     end
   end
 end

data/lib/open_nlp/model.rb

@@ -5,16 +5,20 @@ module OpenNlp
     attr_reader :j_model
 
     def initialize(model)
-      model_stream = case model
-                     when java.io.FileInputStream
-                       model
-                     when String
-                       java.io.FileInputStream.new(model)
-                     else
-                       raise ArgumentError, "Model must be either a string or a java.io.FileInputStream"
-                     end
+      @j_model = self.class.java_class.new(model_stream(model))
+    end
+
+    private
 
-      @j_model = self.class.java_class.new(model_stream)
+    def model_stream(model)
+      case model
+      when java.io.FileInputStream
+        model
+      when String
+        java.io.FileInputStream.new(model)
+      else
+        fail ArgumentError, 'Model must be either a string or a java.io.FileInputStream'
+      end
     end
   end
 end

data/lib/open_nlp/named_entity_detector.rb

@@ -2,9 +2,13 @@ module OpenNlp
   class NamedEntityDetector < Tool
     self.java_class = Java::opennlp.tools.namefind.NameFinderME
 
+    # Detects names for provided array of tokens
+    #
+    # @param [Array<String>] tokens tokens to run name detection on
+    # @return [Array<Java::opennlp.tools.util.Span>] names detected
     def detect(tokens)
-      raise ArgumentError, "tokens must be an instance of Array" unless tokens.is_a?(Array)
-      @j_instance.find(tokens.to_java(:String)).to_ary
+      fail ArgumentError, 'tokens must be an instance of Array' unless tokens.is_a?(Array)
+      j_instance.find(tokens.to_java(:String)).to_ary
     end
   end
 end

data/lib/open_nlp/parser.rb

@@ -1,52 +1,62 @@
 module OpenNlp
   class Parser < Tool
-    def initialize(model, token_model)
-      raise ArgumentError, "model must be an OpenNlp::Model" unless model.is_a?(OpenNlp::Model)
-      raise ArgumentError, "model must be an OpenNlp::Tokenizer::Model" unless token_model.is_a?(Model::Tokenizer)
+    def initialize(parser_model, token_model)
+      unless parser_model.is_a?(OpenNlp::Model)
+        fail ArgumentError, 'parser_model must be an OpenNlp::Model'
+      end
 
-      @j_instance = Java::opennlp.tools.parser.ParserFactory.create(model.j_model)
+      unless token_model.is_a?(Model::Tokenizer)
+        fail ArgumentError, 'token_model must be an OpenNlp::Tokenizer::Model'
+      end
 
+      @j_instance = Java::opennlp.tools.parser.ParserFactory.create(parser_model.j_model)
       @tokenizer = Tokenizer.new(token_model)
     end
 
+    # Parses text into instance of Parse class
+    #
+    # @param [String] text text to parse
+    # @return [OpenNlp::Parser::Parse]
     def parse(text)
-      raise ArgumentError, "str must be a String" unless text.is_a?(String)
-      return {} if text.empty?
-
-      parse_obj = Java::opennlp.tools.parser.Parse.new(
-        text.to_java(:String),
-        Java::opennlp.tools.util.Span.new(0, text.size),
-        Java::opennlp.tools.parser.AbstractBottomUpParser::INC_NODE.to_java(:String),
-        1.to_java(:Double), # probability ?
-        0.to_java(:Integer) # the token index of the head of this parse
-      )
-
-      tokens = @tokenizer.tokenize(text)
-
-      tokens.each_with_index do |tok, i|
-        start = get_token_offset text, tokens, i
-
-        parse_obj.insert Java::opennlp.tools.parser.Parse.new(
-                           text.to_java(:String),
-                           Java::opennlp.tools.util.Span.new(start, start + tok.size),
-                           Java::opennlp.tools.parser.AbstractBottomUpParser::TOK_NODE.to_java(:String),
-                           0.to_java(:Double),
-                           i.to_java(:Integer)
-                         )
-      end
-
-      Parser::Parse.new(@j_instance.parse(parse_obj))
+      raise ArgumentError, 'passed text must be a String' unless text.is_a?(String)
+      text.empty? ? {} : parse_tokens(tokenizer.tokenize(text), text)
     end
 
     private
+
+    attr_reader :tokenizer
+
     def get_token_offset(text, tokens, index)
       offset = 0
+      return offset unless index > 0
 
       for i in (1..index) do
         offset = text.index tokens[i], offset + tokens[i - 1].size
-      end if index > 0
-
+      end
       offset
     end
+
+    def build_parse_obj(text, span_start, span_end, type=Java::opennlp.tools.parser.AbstractBottomUpParser::INC_NODE, probability=1, token_index=0)
+      Java::opennlp.tools.parser.Parse.new(
+        text.to_java(:String),
+        Java::opennlp.tools.util.Span.new(span_start, span_end),
+        type.to_java(:String),
+        probability.to_java(:Double), # probability ?
+        token_index.to_java(:Integer) # the token index of the head of this parse
+      )
+    end
+
+    def parse_tokens(tokens, text)
+      parse_obj = build_parse_obj(text, 0, text.size)
+      parse_type = Java::opennlp.tools.parser.AbstractBottomUpParser::TOK_NODE
+
+      tokens.each_with_index do |tok, i|
+        start = get_token_offset(text, tokens, i)
+        token_parse = build_parse_obj(text, start, start + tok.size, parse_type, 0, i)
+        parse_obj.insert(token_parse)
+      end
+
+      Parser::Parse.new(j_instance.parse(parse_obj))
+    end
   end
-end
+end

data/lib/open_nlp/parser/parse.rb

@@ -13,27 +13,20 @@ module OpenNlp
     end
 
     def tree_bank_string
-      span = j_instance.getSpan
-      text = j_instance.getText
-      type = j_instance.getType
-      start = span.getStart
+      span, text, type, res = j_instance.getSpan, j_instance.getText, j_instance.getType, ''
+      start                 = span.getStart
 
-      res = ''
+      res << "(#{type} " if type != Java::opennlp.tools.parser.AbstractBottomUpParser::TOK_NODE
 
-      res << "(#{type} " unless type == Java::opennlp.tools.parser.AbstractBottomUpParser::TOK_NODE
-
-      j_instance.getChildren.each do |c|
-        s = c.span
-        res << text[start..s.getStart-1] if start < s.getStart
-
-        subtree = self.class.new(c).tree_bank_string
-        res << subtree if subtree
-        start = s.getEnd
+      j_instance.getChildren.each do |child|
+        child_span = child.span
+        res << text[start..child_span.getStart-1] if start < child_span.getStart
+        res << self.class.new(child).tree_bank_string
+        start = child_span.getEnd
       end
 
       res << text[start..span.getEnd-1] if start < span.getEnd
-
-      res << ")" unless type == Java::opennlp.tools.parser.AbstractBottomUpParser::TOK_NODE
+      res << ")" if type != Java::opennlp.tools.parser.AbstractBottomUpParser::TOK_NODE
 
       res
     end
@@ -41,14 +34,12 @@ module OpenNlp
     def code_tree
       kids = j_instance.getChildren
 
-      kids.inject([]) do |acc,kid|
-        data = {:type => kid.getType, :parent_type => self.j_instance.getType, :token => kid.toString}
+      kids.each_with_object([]) do |kid, acc|
+        data    = { :type => kid.getType, :parent_type => self.j_instance.getType, :token => kid.toString }
         subtree = self.class.new(kid).code_tree
         data[:children] = subtree unless subtree.empty?
         acc << data
-
-        acc
       end
     end
   end
-end
+end

data/lib/open_nlp/pos_tagger.rb

@@ -3,8 +3,11 @@ module OpenNlp
     self.java_class = Java::opennlp.tools.postag.POSTaggerME
 
     def tag(tokens)
-      raise ArgumentError, "tokens must be an instance of String or Array" unless (tokens.is_a?(Array) || tokens.is_a?(String))
-      @j_instance.tag(tokens.to_java(:String))
+      unless (tokens.is_a?(Array) || tokens.is_a?(String))
+        fail ArgumentError, 'tokens must be an instance of String or Array'
+      end
+
+      j_instance.tag(tokens.to_java(:String))
     end
   end
 end

data/lib/open_nlp/sentence_detector.rb

@@ -2,14 +2,24 @@ module OpenNlp
   class SentenceDetector < Tool
     self.java_class = Java::opennlp.tools.sentdetect.SentenceDetectorME
 
-    def detect(string)
-      raise ArgumentError, "string must be a String" unless string.is_a?(String)
-      @j_instance.sentDetect(string).to_ary
+    # Detects sentences in a string
+    #
+    # @param [String] string string to detect sentences in
+    # @return [Array<String>] array of detected sentences
+    def detect(str)
+      fail ArgumentError, 'str must be a String' unless str.is_a?(String)
+      j_instance.sentDetect(str).to_ary
     end
 
-    def pos_detect(string)
-      raise ArgumentError, "string must be a String" unless string.is_a?(String)
-      @j_instance.sentPosDetect(string).map{|span| OpenNlp::Util::Span.new(span.getStart, span.getEnd)}
+    # Detects sentences in a string and returns array of spans
+    #
+    # @param [String] str
+    # @return [Array<OpenNlp::Util::Span>] array of spans for detected sentences
+    def pos_detect(str)
+      fail ArgumentError, 'str must be a String' unless str.is_a?(String)
+      j_instance.sentPosDetect(str).map do |span|
+        OpenNlp::Util::Span.new(span.getStart, span.getEnd)
+      end
     end
   end
 end

data/lib/open_nlp/tokenizer.rb

@@ -2,14 +2,19 @@ module OpenNlp
   class Tokenizer < Tool
     self.java_class = Java::opennlp.tools.tokenize.TokenizerME
 
+    # Tokenizes a string
+    #
+    # @param [String] str string to tokenize
+    # @return [Array] array of string tokens
     def tokenize(str)
-      raise ArgumentError, "str must be a String" unless str.is_a?(String)
-      @j_instance.tokenize(str).to_ary
+      fail ArgumentError, 'str must be a String' unless str.is_a?(String)
+      j_instance.tokenize(str).to_ary
     end
 
     private
+
     def get_last_probabilities
-      @j_instance.getTokenProbabilities.to_ary
+      j_instance.getTokenProbabilities.to_ary
     end
   end
 end