rbbt-text 1.2.0 → 1.3.4

data/lib/rbbt/ner/oscar3.rb

@@ -1,7 +1,6 @@
 require 'rbbt'
 require 'rjb'
 require 'libxml'
-require 'rbbt/text/segment'
 require 'rbbt/ner/NER'
 require 'rbbt/util/log'
 
@@ -53,7 +52,7 @@ class OSCAR3 < NER
         next unless type.nil? or type.include? mention_type
         score  = memm ? entities.get(key).to_string.to_f : nil
 
-        NamedEntity.setup mention, rstart.to_i + offset, mention_type, nil, score
+        NamedEntity.setup mention, :offset => rstart.to_i + offset, :entity_type => mention_type, :score => score
         
         mentions << mention unless mentions.collect{|m| m.to_s}.include? mention.to_s
       end

data/lib/rbbt/ner/oscar4.rb

@@ -1,7 +1,7 @@
 require 'rbbt'
 require 'rjb'
 require 'libxml'
-require 'rbbt/text/segment'
+require 'rbbt/segment'
 require 'rbbt/ner/NER'
 require 'rbbt/util/log'
 
@@ -25,7 +25,7 @@ class OSCAR4 < NER
     @@tagger ||= @@OSCAR.new()
   end
 
-  def self.match(text,  type = nil)
+  def self.match(text, protect = false,  type = nil)
     self.init
 
     return [] if text.nil? or text.strip.empty?
@@ -46,7 +46,7 @@ class OSCAR4 < NER
 
       next unless entity.getType.toString == type unless type.nil?
 
-      NamedEntity.setup mention, entity.getStart, entity.getType, inchi, entity.getConfidence
+      NamedEntity.setup mention, :offset => entity.getStart, :entity_type => entity.getType, :code => inchi, :score => entity.getConfidence
 
       result << mention
     end

data/lib/rbbt/ner/patterns.rb

@@ -1,7 +1,7 @@
-require 'rbbt/text/segment/named_entity'
-require 'rbbt/text/segment/segmented'
-require 'rbbt/text/segment/transformed'
-require 'rbbt/text/segment/relationship'
+require 'rbbt/segment/named_entity'
+require 'rbbt/segment/segmented'
+require 'rbbt/segment/transformed'
+#require 'rbbt/segment/relationship'
 require 'rbbt/ner/regexpNER'
 require 'rbbt/ner/token_trieNER'
 require 'rbbt/nlp/nlp'
@@ -14,7 +14,7 @@ class PatternRelExt
     regexpNER = RegExpNER.new type => patterns.collect{|p| /#{p}/}
     segments = sentence.segments
     segments = segments.values.flatten if Hash === segments
-    Transformed.with_transform(sentence, segments, Proc.new{|s| s.type.to_s.upcase}) do |sentence|
+    Transformed.with_transform(sentence, segments, Proc.new{|s| s.entity_type.to_s.upcase}) do |sentence|
       regexpNER.entities(sentence)
     end
   end

data/lib/rbbt/ner/regexpNER.rb

@@ -1,4 +1,3 @@
-require 'rbbt/text/segment'
 require 'rbbt/ner/NER'
 require 'rbbt/util/simpleDSL'
 
@@ -23,7 +22,7 @@ class RegExpNER < NER
       end
 
       if match and not match.empty?
-        NamedEntity.setup(match, start + pre.length, type)
+        NamedEntity.setup(match, :offset => start + pre.length, :entity_type => type)
         matches << match
       end
 

data/lib/rbbt/ner/token_trieNER.rb

@@ -1,46 +1,58 @@
 require 'rbbt'
 require 'rbbt/tsv'
-require 'rbbt/text/segment'
-require 'rbbt/text/segment/token'
+require 'rbbt/segment'
 require 'rbbt/ner/NER'
+require 'rbbt/segment/token'
 
 class TokenTrieNER < NER
-  def self.clean(token)
+  def self.clean(token, stem = false)
     if token.length > 3
-      token.downcase.sub(/-/,'')
+      upcase = token !~ /[a-z]/
+      token = token.downcase.sub(/-/,'')
+
+      if stem && ! upcase
+        require 'stemmer'
+        if stem == :double
+          token = token.stem.stem
+        else
+          token = token.stem
+        end
+      end
+
+      token
     else
       token
     end
   end
 
-  def self.prepare_token(token, start, extend_to_token = true, no_clean = false)
+  def self.prepare_token(token, start, extend_to_token = true, no_clean = false, stem = false)
     if no_clean
       if extend_to_token
-        Token.setup(clean(token), start, token)
+        Token.setup(token, :offset => start, :original => token)
       else
         token
       end
     else
       if extend_to_token
-        Token.setup(clean(token), start, token)
+        Token.setup(clean(token, stem), :offset => start, :original => token)
       else
-        clean(token)
+        clean(token, stem)
       end
     end
   end
 
-  def self.tokenize(text, extend_to_token = true, split_at = nil, no_clean = false, start = 0)
-    split_at = /\s|(\(|\)|[-."':,])/ if split_at.nil?
+  def self.tokenize(text, extend_to_token = true, split_at = nil, no_clean = false, stem = false, start = 0)
+    split_at = /\s|(\(|\)|[-."':,;])/ if split_at.nil?
 
     tokens = []
     while matchdata = text.match(split_at)
-      tokens << prepare_token(matchdata.pre_match, start, extend_to_token, no_clean) unless matchdata.pre_match.empty?
-      tokens << prepare_token(matchdata.captures.first, start + matchdata.begin(1), extend_to_token, no_clean) if matchdata.captures.any? and not matchdata.captures.first.empty?
+      tokens << prepare_token(matchdata.pre_match, start, extend_to_token, no_clean, stem) unless matchdata.pre_match.empty?
+      tokens << prepare_token(matchdata.captures.first, start + matchdata.begin(1), extend_to_token, no_clean, stem) if matchdata.captures.any? and not matchdata.captures.first.empty?
       start += matchdata.end(0)
       text = matchdata.post_match
     end
      
-    tokens << prepare_token(text, start, extend_to_token) unless text.empty?
+    tokens << prepare_token(text, start, extend_to_token, no_clean, stem) unless text.empty?
 
     tokens
   end
@@ -130,14 +142,14 @@ class TokenTrieNER < NER
     index1
   end
 
-  def self.process(index, hash, type = nil, slack = nil, split_at = nil, no_clean = false)
+  def self.process(index, hash, type = nil, slack = nil, split_at = nil, no_clean = false, stem = false)
 
     chunk_size = hash.size / 100
     items_in_chunk = 0
     tmp_index = {}
     hash.send(hash.respond_to?(:through)? :through : :each) do |code, names|
       names = Array === names ? names : [names]
-      names.flatten! if Array === names.first and not Token === names.first.first
+      names.flatten! if Array === names.first and not Segment === names.first.first
 
       if names.empty?
         names.unshift code unless TSV === hash and not (hash.fields.nil? or hash.fields.empty?)
@@ -146,7 +158,7 @@ class TokenTrieNER < NER
       names.each do |name|
         next if name.empty? or (String === name and name.length < 2)
 
-        tokens = Array === name ? name : tokenize(name, false, split_at, no_clean) 
+        tokens = Array === name ? name : tokenize(name, false, split_at, no_clean, stem) 
         tokens.extend EnumeratedArray
 
         token_index = index_for_tokens(tokens, code, type, slack)
@@ -237,10 +249,10 @@ class TokenTrieNER < NER
       match << ((t.respond_to?(:original) and not t.original.nil?) ? t.original : t)
     }
 
-    NamedEntity.setup(match, match_tokens.first.offset, type, codes)
+    NamedEntity.setup(match, :offset => match_tokens.first.offset, :entity_type => type, :code => codes)
   end
 
-  attr_accessor :index, :longest_match, :type, :slack, :split_at, :no_clean
+  attr_accessor :index, :longest_match, :type, :slack, :split_at, :no_clean, :stem
   def initialize(type = nil, file = nil, options = {})
     options = Misc.add_defaults options, :longest_match => true, :no_clean => false, :slack => nil, :split_at => nil,
       :persist => false
@@ -248,6 +260,7 @@ class TokenTrieNER < NER
     @longest_match = options.delete :longest_match
     @split_at = options.delete :split_at
     @no_clean = options.delete :no_clean
+    @stem = options.delete :stem
 
     file = [] if file.nil?
     file = [file] unless Array === file
@@ -273,7 +286,7 @@ class TokenTrieNER < NER
       Log.debug "TokenTrieNER merging TSV"
       new.with_unnamed do
         new.with_monitor({:step => 1000, :desc => "Processing TSV into TokenTrieNER"}) do
-          TokenTrieNER.process(@index, new, type, slack, split_at, no_clean)
+          TokenTrieNER.process(@index, new, type, slack, split_at, no_clean, stem)
         end
       end
     when Hash === new
@@ -284,18 +297,18 @@ class TokenTrieNER < NER
       new = TSV.open(new, :flat)
       new.with_unnamed do
         new.with_monitor({:step => 1000, :desc => "Processing TSV into TokenTrieNER"}) do
-          TokenTrieNER.process(@index, new, type, slack, split_at, no_clean)
+          TokenTrieNER.process(@index, new, type, slack, split_at, no_clean, stem)
         end
       end
     end
   end
 
   def match(text)
-    tokens = Array === text ? text : TokenTrieNER.tokenize(text, true, split_at, no_clean)
+    tokens = Array === text ? text : TokenTrieNER.tokenize(text, true, split_at, no_clean, stem)
 
     tokens.extend EnumeratedArray
     tokens.pos = 0
-
+    
     matches = []
     while tokens.left?
       new_matches = TokenTrieNER.find(@index, tokens, longest_match, slack) 

data/lib/rbbt/nlp/genia/sentence_splitter.rb

@@ -1,5 +1,5 @@
 require 'rbbt/nlp/nlp'
-require 'rbbt/text/segment'
+require 'rbbt/segment'
 module NLP
   Rbbt.claim Rbbt.software.opt.Geniass, :install, Rbbt.share.install.software.Geniass.find
 
@@ -239,6 +239,7 @@ module NLP
   end
 
   def self.geniass_sentence_splitter(text)
+    Rbbt.software.opt.Geniass.produce
     offsets = []
 
     cleaned = text.gsub("\n",NEW_LINE_MASK)
@@ -294,7 +295,7 @@ module NLP
     offsets.collect do |s,e|
       sentence = text[s..e]
       next if sentence.nil?
-      #sentence.gsub!(NEW_LINE_MASK, "\n")
+      sentence.gsub!(NEW_LINE_MASK, "\n")
       Segment.setup sentence, s
       sentence
     end

data/lib/rbbt/nlp/nlp.rb

@@ -2,8 +2,8 @@ require 'rbbt'
 require 'rbbt/util/tmpfile'
 require 'rbbt/persist'
 require 'rbbt/resource'
-require 'rbbt/text/segment'
-require 'rbbt/text/segment/segmented'
+require 'rbbt/segment'
+require 'rbbt/segment/segmented'
 require 'rbbt/nlp/genia/sentence_splitter'
 require 'digest/md5'
 
@@ -101,7 +101,7 @@ module NLP
     input = sentences.collect{|sentence| sentence.gsub(/\n/, NEW_LINE_MASK)} * "\n"
     sentence_tokens = TmpFile.with_file(input) do |fin|
       out = local_persist(Digest::MD5.hexdigest(input), :Chunks, :string) do
-        CMD.cmd("cd #{Rbbt.software.opt.Gdep.find}; ./gdep #{ fin }").read
+        CMD.cmd("cd #{Rbbt.software.opt.Gdep.produce.find}; ./gdep #{ fin }").read
       end
 
       out.split(/^$/).collect do |sentence|
@@ -120,10 +120,10 @@ module NLP
 
 
   def self.gdep_parse_sentences_extension(sentences)
-    require Rbbt.software.opt.Gdep.ruby["Gdep.so"].find
+    require Rbbt.software.opt.Gdep.produce.ruby["Gdep.so"].find
     gdep = Gdep.new
     if not gdep.gdep_is_loaded
-      Misc.in_dir Rbbt.software.opt.Gdep.find do
+      Misc.in_dir Rbbt.software.opt.Gdep.produce.find do
         gdep.load_gdep 
       end
     end

data/lib/rbbt/nlp/open_nlp/sentence_splitter.rb

@@ -1,6 +1,6 @@
 require 'rbbt'
 require 'rjb'
-require 'rbbt/text/segment'
+require 'rbbt/segment'
 require 'rbbt/resource'
 
 module OpenNLP
@@ -33,48 +33,49 @@ module OpenNLP
   def self.sentence_splitter(text)
     return [] if text.nil? or text.empty?
 
-    text = Misc.to_utf8(text)
-    last = 0
-    begin
-      sentence_split_detector = self.sentence_split_detector
-      
-      sentences = nil
-      TmpFile.with_file do |tmpfile|
-        start_time = Time.now
-
-        begin
-          pid = Process.fork do
-            sent = sentence_split_detector.sentDetect(text)
-            Open.write(tmpfile, sent * "#OpenNLP:SENTENCE#")
-          end
+    Segment.ascii(text, "?") do
+      last = 0
+      begin
+        sentence_split_detector = self.sentence_split_detector
 
-          while not Process.waitpid(pid)
-            if Time.now - start_time > MAX
-              Process.kill(9, pid)
-              raise "Taking to long (> #{MAX} seconds)"
-            end
-            sleep 0.1
-          end
+        sentences = nil
+        TmpFile.with_file do |tmpfile|
+          start_time = Time.now
 
           begin
-            Process.waitpid(pid)
+            pid = Process.fork do
+              sent = sentence_split_detector.sentDetect(text)
+              Open.write(tmpfile, sent * "#OpenNLP:SENTENCE#")
+            end
+
+            while not Process.waitpid(pid)
+              if Time.now - start_time > MAX
+                Process.kill(9, pid)
+                raise "Taking to long (> #{MAX} seconds)"
+              end
+              sleep 0.1
+            end
+
+            begin
+              Process.waitpid(pid)
+            end
+          rescue Errno::ECHILD
           end
-        rescue Errno::ECHILD
+
+          sentences = Open.read(tmpfile).split("#OpenNLP:SENTENCE#")
         end
 
-        sentences = Open.read(tmpfile).split("#OpenNLP:SENTENCE#")
+        sentences.collect{|sentence|
+          sentence = Misc.to_utf8(sentence)
+          start = text.index(sentence, last)
+          Segment.setup sentence, start
+          last = start + sentence.length - 1
+          sentence
+        }
+      rescue Exception
+        raise $!
+        raise "Sentence splitter raised exception: #{$!.message}"
       end
-
-      sentences.collect{|sentence|
-        sentence = Misc.to_utf8(sentence)
-        start = text.index(sentence, last)
-        Segment.setup sentence, start
-        last = start + sentence.length - 1
-        sentence
-      }
-    rescue Exception
-      raise $!
-      raise "Sentence splitter raised exception: #{$!.message}"
     end
   end
 end

data/lib/rbbt/nlp/spaCy.rb

@@ -0,0 +1,52 @@
+require 'rbbt/segment'
+require 'rbbt/document'
+require 'rbbt/segment/annotation'
+require 'rbbt/util/python'
+
+module SpaCy
+
+  PROPERTIES = %w(lemma_ is_punct is_space shape_ pos_ tag_)
+
+  def self.tokens(text, lang = 'en')
+
+    tokens = []
+    RbbtPython.run 'spacy' do
+      nlp = spacy.load(lang)
+      doc = nlp.call(text)
+      doc.__len__.times do |i|
+        tokens << doc.__getitem__(i)
+      end
+    end
+    tokens
+  end
+
+  def self.segments(text, lang = 'en')
+    docid = text.docid if Document === text 
+    corpus = text.corpus if Document === text 
+    tokens = self.tokens(text, lang).collect do |token|
+      info = {}
+      PROPERTIES.each do |p|
+        info[p] = token.instance_eval(p.to_s)
+      end
+      info[:type] = "SpaCy"
+      info[:offset] = token.idx
+      info[:dep] = token.dep_ + "->" + token.head.idx.to_s
+      info[:docid] = docid if docid
+      info[:corpus] = corpus if corpus
+      SpaCyToken.setup(token.text, info)
+    end
+    SpaCyToken.setup(tokens, :corpus => corpus)
+  end
+end
+
+module SpaCyToken
+  extend Entity
+  include SegmentAnnotation
+
+  self.annotation *SpaCy::PROPERTIES
+  self.annotation :dep
+end
+
+if __FILE__ == $0
+  ppp Annotated.tsv(SpaCy.segments("I tell a story"), :all)
+end

data/lib/rbbt/segment.rb

@@ -0,0 +1,179 @@
+require 'rbbt-util'
+require 'rbbt/entity'
+require 'rbbt/document'
+
+module SegID
+  extend Entity
+  self.annotation :corpus
+
+  def _parts
+    @parts ||= self.split(":")
+  end
+
+  def range
+    @range ||= Range.new(*_parts[4].split("..").map(&:to_i))
+  end
+
+  def docid
+    @docid ||= DocID.setup(_parts[0..3] * ":")
+  end
+
+  def offset
+    range.begin
+  end
+
+  def segment_length
+    range.end - range.begin + 1
+  end
+
+  property :segment => :single do
+    docid = self.docid
+    document = DocID.setup(docid, :corpus => corpus).document
+
+    text = document[range]
+
+    Segment.setup(text, :docid => docid, :offset => offset)
+  end
+
+  property :segid do
+    self
+  end
+
+end
+
+module Segment
+  extend Entity
+  self.annotation :offset, :docid 
+
+  def segment_length
+    length
+  end
+
+  def eend
+    offset.to_i + length - 1
+  end
+
+  def range
+    (offset.to_i..eend)
+  end
+
+  property :segid do |corpus=nil|
+    SegID.setup([docid, range] * ":", :corpus => corpus)
+  end
+
+  alias id segid
+
+  property :segment do
+    self
+  end
+
+  def self.sort(segments, inline = true)
+    if inline
+      segments.sort do |a,b| 
+        case
+        when ((a.nil? and b.nil?) or (a.offset.nil? and b.offset.nil?))
+          0
+        when (a.nil? or a.offset.nil?)
+          -1
+        when (b.nil? or b.offset.nil?)
+          +1
+        when (not a.range.include? b.offset.to_i and not b.range.include? a.offset.to_i)
+          a.offset.to_i <=> b.offset.to_i
+        else
+          a.segment_length <=> b.segment_length
+        end
+      end
+    else
+      segments.sort_by do |segment| segment.offset.to_i || 0 end.reverse
+    end
+  end
+
+  def self.overlaps(sorted_segments)
+    last = nil
+    overlaped = []
+
+    sorted_segments.reverse.each do |segment| 
+      overlaped << segment if (not last.nil?) and segment.range.end > last 
+      last = segment.range.begin
+    end
+
+    overlaped
+  end
+
+  def self.clean_sort(segments)
+    sorted = sort(segments).reject{|s| s.offset.nil?}
+    overlaps = overlaps(sorted)
+    overlaps.each do |s|
+      sorted.delete s
+    end
+
+    sorted
+  end
+
+  def self.split(text, segments, skip_segments = false)
+    sorted_segments = clean_sort segments
+
+    chunks      = []
+    segment_end = 0
+    text_offset = 0
+    sorted_segments.each do |segment|
+      return chunks if text.nil? or text.empty?
+      next if segment.offset.nil?
+      offset = segment.offset - text_offset
+
+      # Consider segment offset. Save pre, or skip if overlap
+      case
+      when offset < 0 # Overlap, skip
+        next
+      when offset > 0 # Save pre
+        chunk = text[0..offset - 1]
+        Segment.setup(chunk, text_offset)
+        chunks << chunk
+      end
+
+      segment_end = offset + segment.segment_length - 1
+
+      if not skip_segments
+        chunk = text[offset..segment_end]
+        Segment.setup(chunk, text_offset + offset)
+        chunks << chunk
+      end
+
+      text_offset += segment_end + 1
+      text = text[segment_end + 1..-1]
+
+    end
+
+    if not text.nil? and not text.empty?
+      chunk = text.dup
+      Segment.setup(chunk, text_offset)
+      chunks << chunk
+    end
+
+    chunks
+  end
+
+  def self.align(text, parts)
+    pre_offset = 0
+    docid = text.respond_to?(:docid) ? text.docid : nil
+    parts.each do |part|
+      offset = text.index part
+      next if offset.nil?
+      Segment.setup(part, pre_offset + offset, docid)
+      pre_offset += offset + part.segment_length - 1
+      text = text[(offset + part.segment_length - 1)..-1]
+    end
+  end
+
+  def self.index(*args)
+    Segment::RangeIndex.index(*args)
+  end
+
+end
+
+require 'rbbt/segment/range_index'
+require 'rbbt/segment/overlaps'
+require 'rbbt/segment/transformed'
+require 'rbbt/segment/segmented'
+require 'rbbt/segment/encoding'
+