rbbt-text 1.3.0 → 1.3.5

data/lib/rbbt/ner/token_trieNER.rb

@@ -5,15 +5,27 @@ require 'rbbt/ner/NER'
 require 'rbbt/segment/token'
 
 class TokenTrieNER < NER
-  def self.clean(token)
+  def self.clean(token, stem = false)
     if token.length > 3
-      token.downcase.sub(/-/,'')
+      upcase = token !~ /[a-z]/
+      token = token.downcase.sub(/-/,'')
+
+      if stem && ! upcase
+        require 'stemmer'
+        if stem == :double
+          token = token.stem.stem
+        else
+          token = token.stem
+        end
+      end
+
+      token
     else
       token
     end
   end
 
-  def self.prepare_token(token, start, extend_to_token = true, no_clean = false)
+  def self.prepare_token(token, start, extend_to_token = true, no_clean = false, stem = false)
     if no_clean
       if extend_to_token
         Token.setup(token, :offset => start, :original => token)
@@ -22,25 +34,25 @@ class TokenTrieNER < NER
       end
     else
       if extend_to_token
-        Token.setup(clean(token), :offset => start, :original => token)
+        Token.setup(clean(token, stem), :offset => start, :original => token)
       else
-        clean(token)
+        clean(token, stem)
       end
     end
   end
 
-  def self.tokenize(text, extend_to_token = true, split_at = nil, no_clean = false, start = 0)
-    split_at = /\s|(\(|\)|[-."':,])/ if split_at.nil?
+  def self.tokenize(text, extend_to_token = true, split_at = nil, no_clean = false, stem = false, start = 0)
+    split_at = /\s|(\(|\)|[-."':,;])/ if split_at.nil?
 
     tokens = []
     while matchdata = text.match(split_at)
-      tokens << prepare_token(matchdata.pre_match, start, extend_to_token, no_clean) unless matchdata.pre_match.empty?
-      tokens << prepare_token(matchdata.captures.first, start + matchdata.begin(1), extend_to_token, no_clean) if matchdata.captures.any? and not matchdata.captures.first.empty?
+      tokens << prepare_token(matchdata.pre_match, start, extend_to_token, no_clean, stem) unless matchdata.pre_match.empty?
+      tokens << prepare_token(matchdata.captures.first, start + matchdata.begin(1), extend_to_token, no_clean, stem) if matchdata.captures.any? and not matchdata.captures.first.empty?
       start += matchdata.end(0)
       text = matchdata.post_match
     end
      
-    tokens << prepare_token(text, start, extend_to_token) unless text.empty?
+    tokens << prepare_token(text, start, extend_to_token, no_clean, stem) unless text.empty?
 
     tokens
   end
@@ -130,7 +142,7 @@ class TokenTrieNER < NER
     index1
   end
 
-  def self.process(index, hash, type = nil, slack = nil, split_at = nil, no_clean = false)
+  def self.process(index, hash, type = nil, slack = nil, split_at = nil, no_clean = false, stem = false)
 
     chunk_size = hash.size / 100
     items_in_chunk = 0
@@ -146,7 +158,7 @@ class TokenTrieNER < NER
       names.each do |name|
         next if name.empty? or (String === name and name.length < 2)
 
-        tokens = Array === name ? name : tokenize(name, false, split_at, no_clean) 
+        tokens = Array === name ? name : tokenize(name, false, split_at, no_clean, stem) 
         tokens.extend EnumeratedArray
 
         token_index = index_for_tokens(tokens, code, type, slack)
@@ -237,10 +249,11 @@ class TokenTrieNER < NER
       match << ((t.respond_to?(:original) and not t.original.nil?) ? t.original : t)
     }
 
-    NamedEntity.setup(match, :offset => match_tokens.first.offset, :entity_type => type, :code => codes)
+    type = type.first
+    NamedEntity.setup(match, :offset => match_tokens.first.offset, :entity_type => type, :code => codes, :type => type)
   end
 
-  attr_accessor :index, :longest_match, :type, :slack, :split_at, :no_clean
+  attr_accessor :index, :longest_match, :type, :slack, :split_at, :no_clean, :stem
   def initialize(type = nil, file = nil, options = {})
     options = Misc.add_defaults options, :longest_match => true, :no_clean => false, :slack => nil, :split_at => nil,
       :persist => false
@@ -248,6 +261,7 @@ class TokenTrieNER < NER
     @longest_match = options.delete :longest_match
     @split_at = options.delete :split_at
     @no_clean = options.delete :no_clean
+    @stem = options.delete :stem
 
     file = [] if file.nil?
     file = [file] unless Array === file
@@ -273,7 +287,7 @@ class TokenTrieNER < NER
       Log.debug "TokenTrieNER merging TSV"
       new.with_unnamed do
         new.with_monitor({:step => 1000, :desc => "Processing TSV into TokenTrieNER"}) do
-          TokenTrieNER.process(@index, new, type, slack, split_at, no_clean)
+          TokenTrieNER.process(@index, new, type, slack, split_at, no_clean, stem)
         end
       end
     when Hash === new
@@ -284,18 +298,18 @@ class TokenTrieNER < NER
       new = TSV.open(new, :flat)
       new.with_unnamed do
         new.with_monitor({:step => 1000, :desc => "Processing TSV into TokenTrieNER"}) do
-          TokenTrieNER.process(@index, new, type, slack, split_at, no_clean)
+          TokenTrieNER.process(@index, new, type, slack, split_at, no_clean, stem)
         end
       end
     end
   end
 
   def match(text)
-    tokens = Array === text ? text : TokenTrieNER.tokenize(text, true, split_at, no_clean)
+    tokens = Array === text ? text : TokenTrieNER.tokenize(text, true, split_at, no_clean, stem)
 
     tokens.extend EnumeratedArray
     tokens.pos = 0
-
+    
     matches = []
     while tokens.left?
       new_matches = TokenTrieNER.find(@index, tokens, longest_match, slack) 

data/lib/rbbt/nlp/genia/sentence_splitter.rb

@@ -239,6 +239,7 @@ module NLP
   end
 
   def self.geniass_sentence_splitter(text)
+    Rbbt.software.opt.Geniass.produce
     offsets = []
 
     cleaned = text.gsub("\n",NEW_LINE_MASK)
@@ -294,7 +295,7 @@ module NLP
     offsets.collect do |s,e|
       sentence = text[s..e]
       next if sentence.nil?
-      #sentence.gsub!(NEW_LINE_MASK, "\n")
+      sentence.gsub!(NEW_LINE_MASK, "\n")
       Segment.setup sentence, s
       sentence
     end

data/lib/rbbt/nlp/spaCy.rb

@@ -0,0 +1,195 @@
+require 'rbbt/segment'
+require 'rbbt/document'
+require 'rbbt/segment/annotation'
+require 'rbbt/util/python'
+require 'rbbt/network/paths'
+
+module SpaCy
+
+  TOKEN_PROPERTIES = %w(lemma_ is_punct is_space shape_ pos_ tag_)
+  CHUNK_PROPERTIES = %w(lemma_)
+
+  def self.nlp(lang = 'en_core_web_md')
+    @@nlp ||= {}
+    @@nlp[lang] ||= RbbtPython.run :spacy do
+      spacy.load(lang)
+    end
+  end
+
+  def self.tokens(text, lang = 'en_core_web_sm')
+
+    tokens = []
+
+    nlp = nlp(lang)
+    doc = nlp.call(text)
+
+    doc.__len__.times do |i|
+      tokens << doc.__getitem__(i)
+    end
+
+    tokens
+  end
+
+  def self.chunks(text, lang = 'en_core_web_sm')
+
+    tokens = []
+    nlp = nlp(lang)
+
+    doc = nlp.call(text)
+    chunks = doc.noun_chunks.__iter__
+
+    RbbtPython.iterate chunks do |item|
+      tokens << item
+    end
+
+    tokens
+  end
+
+  def self.segments(text, lang = 'en_core_web_sm')
+    docid = text.docid if Document === text
+    corpus = text.corpus if Document === text 
+    tokens = self.tokens(text, lang).collect do |token|
+      info = {}
+      TOKEN_PROPERTIES.each do |p|
+        info[p] = token.instance_eval(p.to_s)
+      end
+      info[:type] = "SpaCy"
+      info[:offset] = token.idx
+      info[:dep] = token.dep_ + "->" + token.head.idx.to_s
+      info[:docid] = docid if docid
+      info[:corpus] = corpus if corpus
+      SpaCyToken.setup(token.text, info)
+    end
+
+    tokens
+  end
+
+  def self.chunk_segments(text, lang = 'en_core_web_sm')
+    docid = text.docid if Document === text
+    corpus = text.corpus if Document === text 
+    chunks = self.chunks(text, lang).collect do |chunk|
+      info = {}
+      CHUNK_PROPERTIES.each do |p|
+        info[p] = chunk.instance_eval(p.to_s)
+      end
+      start = eend =  nil
+      deps = []
+      RbbtPython.iterate chunk.__iter__ do |token|
+        start = token.idx if start.nil?
+        eend = start + chunk.text.length if eend.nil?
+        deps << token.idx.to_s + ":" + token.dep_ + "->" + token.head.idx.to_s if token.head.idx < start || token.head.idx > eend
+      end
+      info[:type] = "SpaCy"
+      info[:offset] = chunk.__iter__.__next__.idx
+      info[:dep] = deps * ";"
+      info[:docid] = docid if docid
+      info[:corpus] = corpus if corpus
+      SpaCySpan.setup(chunk.text, info)
+    end
+
+    chunks
+  end
+
+  def self.dep_graph(text, reverse = false, lang = 'en_core_web_md')
+    tokens = self.segments(text, lang)
+    index = Segment.index(tokens)
+    associations = {}
+    tokens.each do |token|
+      type, target_pos = token.dep.split("->")
+      target_tokens = index[target_pos.to_i]
+      associations[token.segid] = target_tokens
+    end
+
+    if reverse
+      old = associations.dup
+      old.each do |s,ts|
+        ts.each do |t|
+          associations[t] ||= []
+          associations[t] += [s] unless associations[t].include?(s)
+        end
+      end
+    end
+
+    associations
+  end
+
+  def self.chunk_dep_graph(text, reverse = false, lang = 'en_core_web_md')
+    associations = dep_graph(text, false, lang)
+
+    chunks = self.chunk_segments(text, lang)
+    tokens = self.segments(text, lang)
+    index = Segment.index(tokens + chunks)
+
+    chunks.each do |chunk|
+      target_token_ids = chunk.dep.split(";").collect do|dep|
+        type, target_pos = dep.split("->")
+        index[target_pos.to_i]
+      end.flatten
+
+      target_tokens = target_token_ids.collect do |target_token_id|
+        range = Range.new(*target_token_id.split(":").last.split("..").map(&:to_i))
+        range.collect do |pos|
+          index[pos]
+        end.uniq
+      end.flatten
+      associations[chunk.segid] = target_tokens
+    end
+
+    if reverse
+      old = associations.dup
+      old.each do |s,ts|
+        ts.each do |t|
+          associations[t] ||= []
+          associations[t] += [s] unless associations[t].include?(s)
+        end
+      end
+    end
+
+    associations
+  end
+
+  def self.paths(text, source, target, reverse = true, lang = 'en_core_web_md')
+    graph = SpaCy.chunk_dep_graph(text, reverse, lang)
+
+    chunk_index = Segment.index(SpaCy.chunk_segments(text, lang))
+
+    source_id = chunk_index[source.offset].first || source.segid
+    target_id = chunk_index[target.offset].first || target.segid
+
+    path = Paths.dijkstra(graph, source_id, [target_id])
+
+    return nil if path.nil?
+
+    path.reverse
+  end
+
+  def self.config(base, target = nil)
+    TmpFile.with_file(base) do |baseconfig|
+      if target
+        CMD.cmd(:spacy, "init fill-config #{baseconfig} #{target}")
+      else
+        TmpFile.with_file do |tmptarget|
+          CMD.cmd(:spacy, "init fill-config #{baseconfig} #{tmptarget}")
+          Open.read(targetconfig)
+        end
+      end
+    end
+  end
+end
+
+module SpaCyToken
+  extend Entity
+  include SegmentAnnotation
+
+  self.annotation *SpaCy::TOKEN_PROPERTIES
+  self.annotation :dep
+end
+
+module SpaCySpan
+  extend Entity
+  include SegmentAnnotation
+
+  self.annotation *SpaCy::CHUNK_PROPERTIES
+  self.annotation :dep
+end
+

data/lib/rbbt/relationship.rb

@@ -0,0 +1,24 @@
+require 'rbbt/segment'
+
+module Relationship
+  extend Annotation
+  self.annotation :segment
+  self.annotation :terms
+  self.annotation :type
+
+  def text
+    if segment
+      segment
+    else
+      type + ": " + terms * ", "
+    end
+  end
+
+  def html
+    text = <<-EOF
+<span class='Relationship'\
+>#{ self.text }</span>
+    EOF
+    text.chomp
+  end
+end

data/lib/rbbt/segment.rb

@@ -1,5 +1,6 @@
 require 'rbbt-util'
 require 'rbbt/entity'
+require 'rbbt/document'
 
 module SegID
   extend Entity
@@ -10,11 +11,11 @@ module SegID
   end
 
   def range
-    @range ||= Range.new(*_parts.last.split("..").map(&:to_i))
+    @range ||= Range.new(*_parts[4].split("..").map(&:to_i))
   end
 
   def docid
-    @docid ||= _parts[0..3] * ":"
+    @docid ||= DocID.setup(_parts[0..3] * ":")
   end
 
   def offset
@@ -25,12 +26,13 @@ module SegID
     range.end - range.begin + 1
   end
 
-  property :segment do
+  property :segment => :single do
+    docid = self.docid
     document = DocID.setup(docid, :corpus => corpus).document
 
     text = document[range]
 
-    Segment.setup(text, docid)
+    Segment.setup(text, :docid => docid, :offset => offset)
   end
 
   property :segid do
@@ -47,10 +49,13 @@ module Segment
     length
   end
 
+
   def eend
     offset.to_i + length - 1
   end
 
+  alias end eend
+
   def range
     (offset.to_i..eend)
   end

data/lib/rbbt/segment/annotation.rb

@@ -1,6 +1,6 @@
 require 'rbbt-util'
-require 'rbbt/entity'
 require 'rbbt/segment'
+require 'rbbt/entity'
 
 module AnnotID
   extend Entity
@@ -32,7 +32,7 @@ end
 
 module SegmentAnnotation
   extend Entity
-  include Segment
+  include Object::Segment
   self.annotation :type
 
   property :segid do
@@ -47,7 +47,7 @@ module SegmentAnnotation
   end
 
   property :annotid do |corpus=nil|
-    AnnotID.setup([segid, type] * ":", :corpus => corpus)
+    AnnotID.setup([segid, type, Misc.obj2digest(self.info)] * ":", :corpus => corpus)
   end
 
   alias id annotid

data/lib/rbbt/segment/named_entity.rb

@@ -8,6 +8,10 @@ module NamedEntity
 
   self.annotation :entity_type, :code, :score
 
+  def entity_type
+    annotation_values[:entity_type] || annotation_values[:type]
+  end
+
   def report
     <<-EOF
 String: #{ self }
@@ -19,11 +23,14 @@ Score: #{score.inspect}
   end
 
   def html
+    title = code.nil? ? entity_type : [entity_type, code].compact * ":"
+
     text = <<-EOF
 <span class='Entity'\
 #{entity_type.nil? ? "" : " attr-entity-type='#{Array === entity_type ? entity_type * " " : entity_type}'"}\
 #{code.nil?  ? "" : " attr-entity-code='#{Array === code ? code * " " : code}'"}\
 #{score.nil? ? "" : " attr-entity-score='#{Array === score ? score * " " : score}'"}\
+#{title.nil? ? "" : " title='#{Array === title ? title * " " : title}'"}\
 >#{ self }</span>
     EOF
     text.chomp