rbbt-text 1.1.9 → 1.3.3

data/lib/rbbt/ner/linnaeus.rb

@@ -1,12 +1,12 @@
 require 'rjb'
 require 'rbbt'
-require 'rbbt/text/segment/named_entity'
+require 'rbbt/segment/named_entity'
 
 module Linnaeus
 
   Rbbt.claim Rbbt.software.opt.Linnaeus, :install, Rbbt.share.install.software.Linnaeus.find
 
-  ARGS = ["--properties", Rbbt.software.opt.Linnaeus["species-proxy/properties.conf"].find]
+  ARGS = ["--properties", Rbbt.software.opt.Linnaeus.produce["species-proxy/properties.conf"].find]
 
 
   Rjb::load(nil, jvmargs = ['-Xms2G','-Xmx2G']) unless Rjb.loaded?
@@ -31,7 +31,7 @@ module Linnaeus
     init unless defined? @@Matcher
 
     @@Matcher.match(text).toArray().collect do |mention|
-      NamedEntity.setup(mention.text(), mention.start(), "Organism", mention.ids(), mention.probabilities())
+      NamedEntity.setup(mention.text(), :offset => mention.start(), :entity_type => "Organism", :code => mention.ids(), :score => mention.probabilities())
     end
   end
 end

data/lib/rbbt/ner/ngram_prefix_dictionary.rb

@@ -1,8 +1,8 @@
 require 'rbbt'
 require 'rbbt/util/misc'
 require 'rbbt/tsv'
-require 'rbbt/text/segment'
-require 'rbbt/text/segment/token'
+require 'rbbt/segment'
+require 'rbbt/segment/token'
 require 'rbbt/ner/NER'
 require 'inline'
 
@@ -150,7 +150,7 @@ VALUE fast_start_with(VALUE str, VALUE cmp, int offset)
 
   def match(text)
     matches = NGramPrefixDictionary.match(index, (case_insensitive ? text.downcase : text)).collect{|name, code, offset|
-      NamedEntity.setup(name, offset, type, code)
+      NamedEntity.setup(name, :offset => offset, :entity_type => type, :code => code)
     }
 
     if case_insensitive

data/lib/rbbt/ner/oscar3.rb

@@ -1,7 +1,6 @@
 require 'rbbt'
 require 'rjb'
 require 'libxml'
-require 'rbbt/text/segment'
 require 'rbbt/ner/NER'
 require 'rbbt/util/log'
 
@@ -53,7 +52,7 @@ class OSCAR3 < NER
         next unless type.nil? or type.include? mention_type
         score  = memm ? entities.get(key).to_string.to_f : nil
 
-        NamedEntity.setup mention, rstart.to_i + offset, mention_type, nil, score
+        NamedEntity.setup mention, :offset => rstart.to_i + offset, :entity_type => mention_type, :score => score
         
         mentions << mention unless mentions.collect{|m| m.to_s}.include? mention.to_s
       end

data/lib/rbbt/ner/oscar4.rb

@@ -1,7 +1,7 @@
 require 'rbbt'
 require 'rjb'
 require 'libxml'
-require 'rbbt/text/segment'
+require 'rbbt/segment'
 require 'rbbt/ner/NER'
 require 'rbbt/util/log'
 
@@ -25,7 +25,7 @@ class OSCAR4 < NER
     @@tagger ||= @@OSCAR.new()
   end
 
-  def self.match(text,  type = nil)
+  def self.match(text, protect = false,  type = nil)
     self.init
 
     return [] if text.nil? or text.strip.empty?
@@ -46,7 +46,7 @@ class OSCAR4 < NER
 
       next unless entity.getType.toString == type unless type.nil?
 
-      NamedEntity.setup mention, entity.getStart, entity.getType, inchi, entity.getConfidence
+      NamedEntity.setup mention, :offset => entity.getStart, :entity_type => entity.getType, :code => inchi, :score => entity.getConfidence
 
       result << mention
     end

data/lib/rbbt/ner/patterns.rb

@@ -1,7 +1,7 @@
-require 'rbbt/text/segment/named_entity'
-require 'rbbt/text/segment/segmented'
-require 'rbbt/text/segment/transformed'
-require 'rbbt/text/segment/relationship'
+require 'rbbt/segment/named_entity'
+require 'rbbt/segment/segmented'
+require 'rbbt/segment/transformed'
+#require 'rbbt/segment/relationship'
 require 'rbbt/ner/regexpNER'
 require 'rbbt/ner/token_trieNER'
 require 'rbbt/nlp/nlp'
@@ -14,7 +14,7 @@ class PatternRelExt
     regexpNER = RegExpNER.new type => patterns.collect{|p| /#{p}/}
     segments = sentence.segments
     segments = segments.values.flatten if Hash === segments
-    Transformed.with_transform(sentence, segments, Proc.new{|s| s.type.to_s.upcase}) do |sentence|
+    Transformed.with_transform(sentence, segments, Proc.new{|s| s.entity_type.to_s.upcase}) do |sentence|
       regexpNER.entities(sentence)
     end
   end

data/lib/rbbt/ner/regexpNER.rb

@@ -1,4 +1,3 @@
-require 'rbbt/text/segment'
 require 'rbbt/ner/NER'
 require 'rbbt/util/simpleDSL'
 
@@ -23,7 +22,7 @@ class RegExpNER < NER
       end
 
       if match and not match.empty?
-        NamedEntity.setup(match, start + pre.length, type)
+        NamedEntity.setup(match, :offset => start + pre.length, :entity_type => type)
         matches << match
       end
 

data/lib/rbbt/ner/token_trieNER.rb

@@ -1,46 +1,58 @@
 require 'rbbt'
 require 'rbbt/tsv'
-require 'rbbt/text/segment'
-require 'rbbt/text/segment/token'
+require 'rbbt/segment'
 require 'rbbt/ner/NER'
+require 'rbbt/segment/token'
 
 class TokenTrieNER < NER
-  def self.clean(token)
+  def self.clean(token, stem = false)
     if token.length > 3
-      token.downcase.sub(/-/,'')
+      upcase = token !~ /[a-z]/
+      token = token.downcase.sub(/-/,'')
+
+      if stem && ! upcase
+        require 'stemmer'
+        if stem == :double
+          token = token.stem.stem
+        else
+          token = token.stem
+        end
+      end
+
+      token
     else
       token
     end
   end
 
-  def self.prepare_token(token, start, extend_to_token = true, no_clean = false)
+  def self.prepare_token(token, start, extend_to_token = true, no_clean = false, stem = false)
     if no_clean
       if extend_to_token
-        Token.setup(clean(token), start, token)
+        Token.setup(token, :offset => start, :original => token)
       else
         token
       end
     else
       if extend_to_token
-        Token.setup(clean(token), start, token)
+        Token.setup(clean(token, stem), :offset => start, :original => token)
       else
-        clean(token)
+        clean(token, stem)
       end
     end
   end
 
-  def self.tokenize(text, extend_to_token = true, split_at = nil, no_clean = false, start = 0)
-    split_at = /\s|(\(|\)|[-."':,])/ if split_at.nil?
+  def self.tokenize(text, extend_to_token = true, split_at = nil, no_clean = false, stem = false, start = 0)
+    split_at = /\s|(\(|\)|[-."':,;])/ if split_at.nil?
 
     tokens = []
     while matchdata = text.match(split_at)
-      tokens << prepare_token(matchdata.pre_match, start, extend_to_token, no_clean) unless matchdata.pre_match.empty?
-      tokens << prepare_token(matchdata.captures.first, start + matchdata.begin(1), extend_to_token, no_clean) if matchdata.captures.any? and not matchdata.captures.first.empty?
+      tokens << prepare_token(matchdata.pre_match, start, extend_to_token, no_clean, stem) unless matchdata.pre_match.empty?
+      tokens << prepare_token(matchdata.captures.first, start + matchdata.begin(1), extend_to_token, no_clean, stem) if matchdata.captures.any? and not matchdata.captures.first.empty?
       start += matchdata.end(0)
       text = matchdata.post_match
     end
      
-    tokens << prepare_token(text, start, extend_to_token) unless text.empty?
+    tokens << prepare_token(text, start, extend_to_token, no_clean, stem) unless text.empty?
 
     tokens
   end
@@ -130,14 +142,14 @@ class TokenTrieNER < NER
     index1
   end
 
-  def self.process(index, hash, type = nil, slack = nil, split_at = nil, no_clean = false)
+  def self.process(index, hash, type = nil, slack = nil, split_at = nil, no_clean = false, stem = false)
 
     chunk_size = hash.size / 100
     items_in_chunk = 0
     tmp_index = {}
     hash.send(hash.respond_to?(:through)? :through : :each) do |code, names|
       names = Array === names ? names : [names]
-      names.flatten! if Array === names.first and not Token === names.first.first
+      names.flatten! if Array === names.first and not Segment === names.first.first
 
       if names.empty?
         names.unshift code unless TSV === hash and not (hash.fields.nil? or hash.fields.empty?)
@@ -146,7 +158,7 @@ class TokenTrieNER < NER
       names.each do |name|
         next if name.empty? or (String === name and name.length < 2)
 
-        tokens = Array === name ? name : tokenize(name, false, split_at, no_clean) 
+        tokens = Array === name ? name : tokenize(name, false, split_at, no_clean, stem) 
         tokens.extend EnumeratedArray
 
         token_index = index_for_tokens(tokens, code, type, slack)
@@ -237,10 +249,10 @@ class TokenTrieNER < NER
       match << ((t.respond_to?(:original) and not t.original.nil?) ? t.original : t)
     }
 
-    NamedEntity.setup(match, match_tokens.first.offset, type, codes)
+    NamedEntity.setup(match, :offset => match_tokens.first.offset, :entity_type => type, :code => codes)
   end
 
-  attr_accessor :index, :longest_match, :type, :slack, :split_at, :no_clean
+  attr_accessor :index, :longest_match, :type, :slack, :split_at, :no_clean, :stem
   def initialize(type = nil, file = nil, options = {})
     options = Misc.add_defaults options, :longest_match => true, :no_clean => false, :slack => nil, :split_at => nil,
       :persist => false
@@ -248,6 +260,7 @@ class TokenTrieNER < NER
     @longest_match = options.delete :longest_match
     @split_at = options.delete :split_at
     @no_clean = options.delete :no_clean
+    @stem = options.delete :stem
 
     file = [] if file.nil?
     file = [file] unless Array === file
@@ -273,7 +286,7 @@ class TokenTrieNER < NER
       Log.debug "TokenTrieNER merging TSV"
       new.with_unnamed do
         new.with_monitor({:step => 1000, :desc => "Processing TSV into TokenTrieNER"}) do
-          TokenTrieNER.process(@index, new, type, slack, split_at, no_clean)
+          TokenTrieNER.process(@index, new, type, slack, split_at, no_clean, stem)
         end
       end
     when Hash === new
@@ -284,18 +297,18 @@ class TokenTrieNER < NER
       new = TSV.open(new, :flat)
       new.with_unnamed do
         new.with_monitor({:step => 1000, :desc => "Processing TSV into TokenTrieNER"}) do
-          TokenTrieNER.process(@index, new, type, slack, split_at, no_clean)
+          TokenTrieNER.process(@index, new, type, slack, split_at, no_clean, stem)
         end
       end
     end
   end
 
   def match(text)
-    tokens = Array === text ? text : TokenTrieNER.tokenize(text, true, split_at, no_clean)
+    tokens = Array === text ? text : TokenTrieNER.tokenize(text, true, split_at, no_clean, stem)
 
     tokens.extend EnumeratedArray
     tokens.pos = 0
-
+    
     matches = []
     while tokens.left?
       new_matches = TokenTrieNER.find(@index, tokens, longest_match, slack) 

data/lib/rbbt/nlp/genia/sentence_splitter.rb

@@ -1,5 +1,5 @@
 require 'rbbt/nlp/nlp'
-require 'rbbt/text/segment'
+require 'rbbt/segment'
 module NLP
   Rbbt.claim Rbbt.software.opt.Geniass, :install, Rbbt.share.install.software.Geniass.find
 
@@ -239,6 +239,7 @@ module NLP
   end
 
   def self.geniass_sentence_splitter(text)
+    Rbbt.software.opt.Geniass.produce
     offsets = []
 
     cleaned = text.gsub("\n",NEW_LINE_MASK)
@@ -294,7 +295,7 @@ module NLP
     offsets.collect do |s,e|
       sentence = text[s..e]
       next if sentence.nil?
-      #sentence.gsub!(NEW_LINE_MASK, "\n")
+      sentence.gsub!(NEW_LINE_MASK, "\n")
       Segment.setup sentence, s
       sentence
     end

data/lib/rbbt/nlp/nlp.rb

@@ -2,8 +2,8 @@ require 'rbbt'
 require 'rbbt/util/tmpfile'
 require 'rbbt/persist'
 require 'rbbt/resource'
-require 'rbbt/text/segment'
-require 'rbbt/text/segment/segmented'
+require 'rbbt/segment'
+require 'rbbt/segment/segmented'
 require 'rbbt/nlp/genia/sentence_splitter'
 require 'digest/md5'
 
@@ -101,7 +101,7 @@ module NLP
     input = sentences.collect{|sentence| sentence.gsub(/\n/, NEW_LINE_MASK)} * "\n"
     sentence_tokens = TmpFile.with_file(input) do |fin|
       out = local_persist(Digest::MD5.hexdigest(input), :Chunks, :string) do
-        CMD.cmd("cd #{Rbbt.software.opt.Gdep.find}; ./gdep #{ fin }").read
+        CMD.cmd("cd #{Rbbt.software.opt.Gdep.produce.find}; ./gdep #{ fin }").read
       end
 
       out.split(/^$/).collect do |sentence|
@@ -120,10 +120,10 @@ module NLP
 
 
   def self.gdep_parse_sentences_extension(sentences)
-    require Rbbt.software.opt.Gdep.ruby["Gdep.so"].find
+    require Rbbt.software.opt.Gdep.produce.ruby["Gdep.so"].find
     gdep = Gdep.new
     if not gdep.gdep_is_loaded
-      Misc.in_dir Rbbt.software.opt.Gdep.find do
+      Misc.in_dir Rbbt.software.opt.Gdep.produce.find do
         gdep.load_gdep 
       end
     end

data/lib/rbbt/nlp/open_nlp/sentence_splitter.rb

@@ -1,6 +1,6 @@
 require 'rbbt'
 require 'rjb'
-require 'rbbt/text/segment'
+require 'rbbt/segment'
 require 'rbbt/resource'
 
 module OpenNLP
@@ -33,48 +33,49 @@ module OpenNLP
   def self.sentence_splitter(text)
     return [] if text.nil? or text.empty?
 
-    text = Misc.to_utf8(text)
-    last = 0
-    begin
-      sentence_split_detector = self.sentence_split_detector
-      
-      sentences = nil
-      TmpFile.with_file do |tmpfile|
-        start_time = Time.now
-
-        begin
-          pid = Process.fork do
-            sent = sentence_split_detector.sentDetect(text)
-            Open.write(tmpfile, sent * "#OpenNLP:SENTENCE#")
-          end
+    Segment.ascii(text, "?") do
+      last = 0
+      begin
+        sentence_split_detector = self.sentence_split_detector
 
-          while not Process.waitpid(pid)
-            if Time.now - start_time > MAX
-              Process.kill(9, pid)
-              raise "Taking to long (> #{MAX} seconds)"
-            end
-            sleep 0.1
-          end
+        sentences = nil
+        TmpFile.with_file do |tmpfile|
+          start_time = Time.now
 
           begin
-            Process.waitpid(pid)
+            pid = Process.fork do
+              sent = sentence_split_detector.sentDetect(text)
+              Open.write(tmpfile, sent * "#OpenNLP:SENTENCE#")
+            end
+
+            while not Process.waitpid(pid)
+              if Time.now - start_time > MAX
+                Process.kill(9, pid)
+                raise "Taking to long (> #{MAX} seconds)"
+              end
+              sleep 0.1
+            end
+
+            begin
+              Process.waitpid(pid)
+            end
+          rescue Errno::ECHILD
           end
-        rescue Errno::ECHILD
+
+          sentences = Open.read(tmpfile).split("#OpenNLP:SENTENCE#")
         end
 
-        sentences = Open.read(tmpfile).split("#OpenNLP:SENTENCE#")
+        sentences.collect{|sentence|
+          sentence = Misc.to_utf8(sentence)
+          start = text.index(sentence, last)
+          Segment.setup sentence, start
+          last = start + sentence.length - 1
+          sentence
+        }
+      rescue Exception
+        raise $!
+        raise "Sentence splitter raised exception: #{$!.message}"
       end
-
-      sentences.collect{|sentence|
-        sentence = Misc.to_utf8(sentence)
-        start = text.index(sentence, last)
-        Segment.setup sentence, start
-        last = start + sentence.length - 1
-        sentence
-      }
-    rescue Exception
-      raise $!
-      raise "Sentence splitter raised exception: #{$!.message}"
     end
   end
 end

data/lib/rbbt/nlp/spaCy.rb

@@ -0,0 +1,52 @@
+require 'rbbt/segment'
+require 'rbbt/document'
+require 'rbbt/segment/annotation'
+require 'rbbt/util/python'
+
+module SpaCy
+
+  PROPERTIES = %w(lemma_ is_punct is_space shape_ pos_ tag_)
+
+  def self.tokens(text, lang = 'en')
+
+    tokens = []
+    RbbtPython.run 'spacy' do
+      nlp = spacy.load(lang)
+      doc = nlp.call(text)
+      doc.__len__.times do |i|
+        tokens << doc.__getitem__(i)
+      end
+    end
+    tokens
+  end
+
+  def self.segments(text, lang = 'en')
+    docid = text.docid if Document === text 
+    corpus = text.corpus if Document === text 
+    tokens = self.tokens(text, lang).collect do |token|
+      info = {}
+      PROPERTIES.each do |p|
+        info[p] = token.instance_eval(p.to_s)
+      end
+      info[:type] = "SpaCy"
+      info[:offset] = token.idx
+      info[:dep] = token.dep_ + "->" + token.head.idx.to_s
+      info[:docid] = docid if docid
+      info[:corpus] = corpus if corpus
+      SpaCyToken.setup(token.text, info)
+    end
+    SpaCyToken.setup(tokens, :corpus => corpus)
+  end
+end
+
+module SpaCyToken
+  extend Entity
+  include SegmentAnnotation
+
+  self.annotation *SpaCy::PROPERTIES
+  self.annotation :dep
+end
+
+if __FILE__ == $0
+  ppp Annotated.tsv(SpaCy.segments("I tell a story"), :all)
+end