rbbt-text 0.2.1 → 0.5.0

data/lib/rbbt/ner/annotations/annotated.rb

@@ -0,0 +1,15 @@
+require 'rbbt/ner/annotations'
+module Annotated
+  attr_accessor :annotations
+  def self.annotate(string, annotations = nil)
+    string.extend Annotated
+    string.annotations = annotations || []
+    string
+  end
+
+  def split_segments(skip_segments = false)
+    Segment.split(self, @annotations, skip_segments)
+  end
+end
+
+

data/lib/rbbt/ner/annotations/named_entity.rb

@@ -0,0 +1,37 @@
+require 'rbbt/ner/annotations'
+
+module NamedEntity 
+  attr_accessor :type, :code, :score, :segment_types
+  include Segment
+
+  def self.annotate(string, offset = nil, type = nil, code = nil, score = nil)
+    string.extend NamedEntity
+    string.offset = offset unless offset.nil?
+    string.type  = type unless type.nil?
+    string.code  = code unless code.nil?
+    string.score = score unless score.nil?
+    string
+  end
+
+  def report
+    <<-EOF
+String: #{ self }
+Offset: #{ offset.inspect }
+Type: #{type.inspect}
+Code: #{code.inspect}
+Score: #{score.inspect}
+    EOF
+  end
+
+  def html
+    text = <<-EOF
+<span class='Entity'\
+#{type.nil? ? "" : " attr-entity-type='#{Array === type ? type * " " : type}'"}\
+#{code.nil?  ? "" : " attr-entity-code='#{Array === code ? code * " " : code}'"}\
+#{score.nil? ? "" : " attr-entity-score='#{Array === score ? score * " " : score}'"}\
+>#{ self }</span>
+    EOF
+    text.chomp
+  end
+end
+

data/lib/rbbt/ner/annotations/relations.rb

@@ -0,0 +1,25 @@
+require 'rbbt/ner/annotations'
+
+module Relationship
+  attr_accessor :terms, :segment_types
+  include Segment
+  def self.annotate(string, offset = nil, terms = nil)
+    string.extend PPI
+    string.offset = offset unless offset.nil?
+    string.terms = terms unless terms.nil?
+    string
+  end
+
+  def html
+    text = <<-EOF
+<span class='Relationship'\
+>#{ self }</span>
+    EOF
+    text.chomp
+  end
+
+  def html_with_entities(*types)
+    annotations.values_at(*types).each do |segments|
+    end
+  end
+end

data/lib/rbbt/ner/annotations/token.rb

@@ -0,0 +1,28 @@
+require 'rbbt/ner/annotations'
+
+module Token
+  include Segment
+  attr_accessor :original
+  def self.annotate(string, offset = nil, original = nil)
+    string.extend Token
+    string.offset   = offset unless offset.nil?
+    string.original = original || string.dup
+    string
+  end
+
+  def self.tokenize(text, split_at = /\s|(\(|\)|[-."':,])/, start = 0)
+
+    tokens = []
+    while matchdata = text.match(split_at)
+      tokens << Token.annotate(matchdata.pre_match, start) unless matchdata.pre_match.empty?
+      tokens << Token.annotate(matchdata.captures.first, start + matchdata.begin(1)) if matchdata.captures.any? and not matchdata.captures.first.empty?
+      start += matchdata.end(0)
+      text = matchdata.post_match
+    end
+
+    tokens << Token.annotate(text, start) unless text.empty?
+
+    tokens
+  end
+end
+

data/lib/rbbt/ner/annotations/transformed.rb

@@ -0,0 +1,170 @@
+require 'rbbt/ner/annotations'
+module Transformed
+  attr_accessor :transformation_offset_differences, :transformation_original
+
+  def self.with_transform(text, segments, replacement)
+    require 'rbbt/util/misc'
+
+    text.extend Transformed
+    text.replace(segments, replacement)
+
+    segments = yield text
+
+    segments = nil unless Array === segments
+
+    text.restore(segments, true)
+  end
+ 
+  def self.transform(text, segments, replacement = nil, &block)
+    require 'rbbt/util/misc'
+
+    text.extend Transformed
+    text.replace(segments, replacement, &block)
+
+    text
+  end
+ 
+  def transform_pos(pos)
+    return pos if transformation_offset_differences.nil?
+    # tranformation_offset_differences are assumed to be sorted in reverse
+    # order
+    transformation_offset_differences.reverse.each do |trans_diff|
+      acc = 0
+      trans_diff.reverse.each do |offset, diff, orig_length, trans_length|
+        break if offset >=  pos
+        acc += diff 
+      end
+      pos = pos - acc 
+    end
+
+    pos
+  end
+
+  def transform_range(range)
+    (transform_pos(range.begin)..transform_pos(range.end))
+  end
+
+  def transformed_set(pos, value)
+    transformed_pos = case
+                when Range === pos
+                  transform_range(pos)
+                when Integer === pos
+                  transform_pos(pos)
+                else
+                  raise "Text position not understood '#{pos.inspect}'. Not Range or Integer"
+                end
+
+    self[transformed_pos] = value
+  end
+
+  def transformed_get(pos)
+    transformed_pos = case
+                when Range === pos
+                  transform_range(pos)
+                when Integer === pos
+                  transform_pos(pos)
+                else
+                  raise "Text position not understood '#{pos.inspect}'. Not Range or Integer"
+                end
+
+    self[transformed_pos]
+  end
+
+  def conflict?(segment_range)
+    return false if @transformation_offset_differences.nil? or @transformation_offset_differences.empty?
+    transformation_offset_difference = @transformation_offset_differences.last
+
+    transformation_offset_difference.each do |info|
+      offset, diff, orig_length, trans_length = info
+      return true if segment_range.begin > offset and segment_range.begin < offset + trans_length or
+      segment_range.end   > offset and segment_range.end   < offset + trans_length
+    end
+
+    return false
+  end
+
+  def replace(segments, replacement = nil, &block)
+    replacement ||= block
+    raise "No replacement given" if replacement.nil?
+    transformation_offset_differences = []
+    transformation_original = []
+
+    Segment.clean_sort(segments).reverse.each do |segment|
+      untransformed_segment_range_here= segment.range_in(self)
+      transformed_segment_range  = self.transform_range(untransformed_segment_range_here)
+      next if conflict?(transformed_segment_range)
+
+      text_before_transform = self[transformed_segment_range]
+
+      case
+      when String === replacement
+        transformed_text = replacement
+      when Proc === replacement
+
+        # Prepare segment with new text
+        save_segment_text = segment.dup
+        save_offset = segment.offset
+        segment.replace text_before_transform
+        segment.offset = transformed_segment_range.begin
+
+        transformed_text = replacement.call segment
+
+        # Restore segment with original text
+        segment.replace save_segment_text
+        segment.offset = save_offset
+      else
+        raise "Replacemente not String nor Proc"
+      end
+      diff = segment.length - transformed_text.length
+      self[transformed_segment_range] = transformed_text
+
+      transformation_offset_differences << [untransformed_segment_range_here.begin, diff, text_before_transform.length, transformed_text.length]
+      transformation_original << text_before_transform
+    end
+
+    @transformation_offset_differences ||= []
+    @transformation_offset_differences << transformation_offset_differences
+    @transformation_original ||= []
+    @transformation_original << transformation_original
+  end
+
+  def restore(segments = nil, first_only = false)
+    stop = false
+    while self.transformation_offset_differences.any? and not stop
+      transformation_offset_differences = self.transformation_offset_differences.pop
+      transformation_original           = self.transformation_original.pop
+
+      ranges = transformation_offset_differences.collect do |offset,diff,orig_length,rep_length|
+        (offset..(offset + rep_length - 1))
+      end
+
+      ranges.zip(transformation_original).reverse.each do |range,text|
+        self.transformed_set(range, text)
+      end
+
+      stop = true if first_only
+
+      next if segments.nil?
+
+      segment_ranges = segments.each do |segment|
+        r = segment.range
+
+        s = r.begin
+        e = r.end
+        sdiff = 0
+        ediff = 0
+        transformation_offset_differences.reverse.each do |offset,diff,orig_length,rep_length|
+          sdiff += diff if offset < s
+          ediff += diff if offset + rep_length - 1 < e
+        end
+
+        segment.offset = s + sdiff
+        segment.replace self[(s+sdiff)..(e + ediff)]
+      end
+    end
+
+    segments
+  end
+end
+
+

data/lib/rbbt/ner/banner.rb

@@ -7,7 +7,7 @@ require 'rbbt/ner/NER'
 # in Java. Banner[http://banner.sourceforge.net/].
 class Banner < NER
 
-  Rbbt.add_software "BANNER" => ['','']
+  Rbbt.software.opt.BANNER.define_as_install Rbbt.share.install.software.BANNER.find
 
   @@JFile = Rjb::import('java.io.File')
   @@SimpleTokenizer = Rjb::import('banner.tokenization.SimpleTokenizer')
@@ -22,9 +22,9 @@ class Banner < NER
   # The parameters are set to default values, the only one that one
   # might want to change is the modelfile to point to a custom trained
   # one.
-  def initialize(modelfile = File.join(Rbbt.find_software('BANNER'), 'gene_model.bin'),
-                 lemmadir  = File.join(Rbbt.find_software('BANNER'), 'nlpdata/lemmatiser'),
-                 taggerdir = File.join(Rbbt.find_software('BANNER'), 'nlpdata/tagger')
+  def initialize(modelfile = Rbbt.software.opt.BANNER["gene_model.bin"].find,
+                 lemmadir  = Rbbt.software.opt.BANNER.nlpdata.lemmatiser.find,
+                 taggerdir = Rbbt.software.opt.BANNER.nlpdata.tagger.find
                 )
 
     @tokenizer = @@SimpleTokenizer.new
@@ -50,8 +50,10 @@ class Banner < NER
   # Returns an array with the mention found in the provided piece of
   # text.
   def match(text)
+    return [] if text.nil? 
     text.gsub!(/\n/,' ')
     text.gsub!(/\|/,'/') # Character | gives an error
+    return [] if text.strip.empty? 
     sentence = @@Sentence.new(text)
 
     @tokenizer.tokenize(sentence)
@@ -65,7 +67,8 @@ class Banner < NER
       mention = $1
       mention.sub!(/^\s*/,'')
       mention.sub!(/\s*$/,'')
-      NamedEntity.annotate mention, nil, 'GENE'
+      offset = text.index(mention)
+      NamedEntity.annotate(mention, offset, 'GENE')
       mention
     }
     res

data/lib/rbbt/ner/chemical_tagger.rb

@@ -0,0 +1,34 @@
+require 'rbbt'
+require 'rjb'
+require 'rbbt/ner/annotations'
+require 'rbbt/ner/NER'
+require 'rbbt/util/log'
+
+class ChemicalTagger < NER
+  Rbbt.software.opt.ChemicalTagger.define_as_install Rbbt.share.install.software.ChemicalTagger.find
+
+  Rjb::load(nil, jvmargs = ['-Xms128m','-Xmx2048m'])
+
+  @@RbbtChemicalTagger = Rjb::import('RbbtChemicalTagger')
+
+  def self.match(text,  type = nil, memm =  false)
+
+    return [] if text.nil? or text.strip.empty?
+
+    begin
+      matches = @@RbbtChemicalTagger.match(text)
+    rescue
+      Log.debug "ChemicalTagger Error: #{$!.message}"
+      return []
+    end
+
+    matches.collect do |mention|
+      offset = text.index mention
+      NamedEntity.annotate mention, offset, "Chemical Mention", nil, nil
+    end
+  end
+
+  def match(*args)
+    ChemicalTagger.match(*args)
+  end
+end

data/lib/rbbt/ner/ngram_prefix_dictionary.rb

@@ -0,0 +1,136 @@
+require 'rbbt-util'
+require 'rbbt/util/tsv'
+require 'rbbt/ner/annotations'
+require 'rbbt/ner/annotations/token'
+require 'rbbt/ner/NER'
+require 'inline'
+
+# This code was adapted from Ashish Tendulkar (ASK MARTIN)
+class NGramPrefixDictionary < NER
+  STOP_LETTERS = %w(\' " ( ) { } [ ] - ? ! < ; : > . ,)
+  STOP_LETTER_CHAR_VALUES = STOP_LETTERS.collect{|l| l[0]}
+  class << self
+    inline do |builder|
+
+      builder.c_raw <<-EOC
+int is_stop_letter(char letter)
+{
+
+  if( letter == ' ' || #{STOP_LETTERS.collect{|l| "letter == '#{l}' "} * "||"} ){ return 1;}
+
+  return 0;
+}
+      EOC
+
+      builder.c <<-EOC
+VALUE fast_start_with(VALUE str, VALUE cmp, int offset)
+{
+  int length_cmp = RSTRING_LEN(cmp);
+  int length_str = RSTRING_LEN(str);
+
+  if (memcmp(RSTRING_PTR(str)+ offset, RSTRING_PTR(cmp), length_cmp) == 0){
+    if (length_cmp - offset == length_str || is_stop_letter(RSTRING_PTR(str)[offset + length_cmp]))
+      return Qtrue;
+    else
+      return Qfalse;
+  }
+
+  return Qfalse;
+}
+      EOC
+    end
+  end
+
+  def self.process(hash)
+    index = {}
+    hash.each do |code, names|
+      names.each do |name|
+        ngram = name[0..2].strip
+        index[ngram] ||= []
+        index[ngram] << [name, code]
+      end
+    end
+    index
+  end
+
+  def self.match(index, text)
+    matches = []
+
+    text_offset = 0
+    text_length = text.length
+    while (not text_offset.nil?) and text_offset < text_length
+      text_offset += 1 if STOP_LETTER_CHAR_VALUES.include? text[text_offset]
+      ngram =  text[text_offset..text_offset + 2].strip
+
+      found = nil
+      if index.include? ngram
+
+        diff = text_length - text_offset
+        # Match with entries
+        index[ngram].each do |name, code|
+          if name.length < diff
+            #if piece.start_with? name and 
+            #  (text_offset + name.length == text_length or piece[name.length] == " "[0])
+
+            if fast_start_with(text, name, text_offset)
+              found = [name, code, text_offset]
+              break
+            end
+          end
+        end
+      end
+
+      if found.nil?
+        text_offset = text.index(" ", text_offset)
+        text_offset += 1 unless text_offset.nil?
+      else
+        matches << found
+        text_offset += found.first.length
+      end
+    end
+
+    matches
+  end
+
+  attr_accessor :index, :type
+  def initialize(file, type = nil)
+    tsv = TSV.new(file, :flat)
+    @type = type
+    tsv.unnamed = true
+    @index = NGramPrefixDictionary.process(tsv)
+  end
+
+  def match(text)
+    NGramPrefixDictionary.match(index, text).collect{|name, code, offset|
+      NamedEntity.annotate(name, offset, type, code)
+    }
+  end
+end
+
+if __FILE__ == $0
+  require 'rbbt/sources/jochem'
+  require 'rbbt/sources/pubmed'
+
+  texts = []
+  index = {}
+
+  texts = PubMed.get_article(PubMed.query("GB-1a", 100)).collect do |pmid, article|
+    article.text
+  end
+
+  texts *= 150/texts.length
+
+  tsv = Rbbt.share.databases.JoChem.lexicon.tsv :flat, :persistence => false, :grep => "GB"
+  #tsv = Rbbt.share.databases.JoChem.lexicon.tsv :flat, :persistence => true
+
+  tsv.unnamed = true
+  ner = NGramPrefixDictionary.new(tsv)
+
+  Misc.benchmark do
+    texts.each do |text|
+      ner.match(text)
+    end
+  end
+
+
+end