rbbt-text 0.2.0 → 0.2.1

data/lib/rbbt/ner/token_trieNER.rb

@@ -0,0 +1,168 @@
+require 'rbbt-util'
+require 'rbbt/util/tsv'
+require 'rbbt/ner/annotations'
+require 'rbbt/ner/NER'
+
+class TokenTrieNER < NER
+  def self.clean(token)
+    if token.length > 3
+      token.downcase
+    else
+      token
+    end
+  end
+
+  def self.prepare_token(token, start)
+    Token.annotate(clean(token), start, token)
+  end
+
+  def self.tokenize(text, split_at = /\s|(\(|\)|[-."':,])/, start = 0)
+
+    tokens = []
+    while matchdata = text.match(split_at)
+      tokens << prepare_token(matchdata.pre_match, start) unless matchdata.pre_match.empty?
+      tokens << prepare_token(matchdata.captures.first, start + matchdata.begin(1)) if matchdata.captures.any? and not matchdata.captures.first.empty?
+      start += matchdata.end(0)
+      text = matchdata.post_match
+    end
+    tokens << prepare_token(text, start) unless text.empty?
+
+    tokens
+  end
+
+  #{{{ Process dictionary
+
+  class Code
+    attr_accessor :value, :type
+    def initialize(value, type = nil)
+      @value = value
+      @type = type
+    end
+
+    def to_s
+      [type, value] * ":"
+    end
+  end
+
+  def self.index_for_tokens(tokens, code, type = nil)
+    if tokens.empty?
+      {:END => [Code.new code, type]}
+    else
+      {tokens.shift => index_for_tokens(tokens, code, type)}
+    end
+  end
+  
+  def self.merge(index1, index2)
+    index2.each do |key, new_index2|
+      case
+      when key == :END
+        index1[:END] ||= []
+        index1[:END] += new_index2.reject{|new| index1[:END].collect{|e| e.to_s }.include? new.to_s }
+        index1[:END].uniq!
+      when index1.include?(key)
+        merge(index1[key], new_index2)
+      else
+        index1[key] = new_index2
+      end
+    end
+  end
+
+  def self.process(hash, type = nil)
+    index = {}
+    hash.each do |code, names|
+      names.flatten.each do |name|
+        next if name.empty? or name.length < 2
+        tokens = tokenize name
+
+        merge(index, index_for_tokens(tokens, code, type)) unless tokens.empty?
+      end
+    end
+    index
+  end
+
+  #{{{ Matching
+  
+  def self.find(index, tokens, longest_match = true)
+    return nil unless index.include? tokens.first
+
+    head = tokens.shift
+    next_index = index[head]
+
+    if tokens.empty?
+      if next_index.include? :END
+        return [next_index[:END], [head]]
+      else
+        tokens.unshift head
+        return nil
+      end
+    else
+
+      return [next_index[:END], [head]] if next_index.include?(:END) and not longest_match
+
+      matches = find(next_index, tokens)
+      if not matches.nil?
+        matches.last.unshift head
+        return matches
+      end
+      
+      return [next_index[:END], [head]] if next_index.include?(:END)
+
+      tokens.unshift head
+      return nil
+    end
+  end
+
+  def self.make_match(match_tokens, type, codes)
+    match = ""
+    match_offset = match_tokens.first.offset
+    match_tokens.each{|t| 
+      match << " " * (t.offset - (match_offset + match.length)) if t.offset > (match_offset + match.length)
+      match << t.original
+    }
+
+    NamedEntity.annotate(match, match_tokens.first.offset, type, codes)
+  end
+
+  attr_accessor :index, :longest_match, :type
+  def initialize(file, type = nil, options = {})
+    options = Misc.add_defaults options, :flatten => true, :longest_match => true
+    @longest_match = options.delete :longest_match
+
+    file = [file] unless Array === file
+    @index = {}
+    file.each do |f| TokenTrieNER.merge(@index, TokenTrieNER.process(TSV.new(f, options), type)) end
+  end
+
+  def merge(new, type = nil)
+    case
+    when TokenTrieNER === new
+      TokenTrieNER.merge(@index, new.index)
+    when Hash === new
+      TokenTrieNER.merge(@index, new)
+    when TSV === new
+      TokenTrieNER.merge(@index, TokenTrieNER.process(new,type))
+    when String === new
+      TokenTrieNER.merge(@index, TokenTrieNER.process(TSV.new(new, :flatten => true), type))
+    end
+  end
+
+  def match(text)
+    tokens = TokenTrieNER.tokenize text
+
+    matches = []
+    while tokens.any?
+      new_matches = TokenTrieNER.find(@index, tokens, longest_match) 
+
+      if new_matches
+        codes, match_tokens = new_matches
+        matches << TokenTrieNER.make_match(match_tokens, codes.collect{|c| c.type}, codes.collect{|c| c.value})
+      else
+        tokens.shift
+      end
+    end
+
+    matches
+  end
+
+end
+

data/test/rbbt/ner/test_NER.rb

@@ -0,0 +1,10 @@
+require File.expand_path(File.dirname(__FILE__)) + '/../../test_helper.rb'
+require 'rbbt/util/tmpfile'
+require 'rbbt/ner/NER'
+
+class TestNER < Test::Unit::TestCase
+  def test_true
+    assert true
+  end
+end
+

data/test/rbbt/ner/test_abner.rb

@@ -4,11 +4,11 @@ require 'test/unit'
 
 class TestAbner < Test::Unit::TestCase
 
-  def test_extract
+  def test_match
     begin
       ner = Abner.new
 
-      mentions = ner.extract(" The P-ITIM-compelled multi-phosphoprotein complex binds to and activates SHP-2, which in turn dephosphorylates SHIP and Shc and probably other substrates.")
+      mentions = ner.match(" The P-ITIM-compelled multi-phosphoprotein complex binds to and activates SHP-2, which in turn dephosphorylates SHIP and Shc and probably other substrates.")
       ["SHP-2", "SHIP", "Shc"].each{|mention| 
         assert(mentions.include? mention)
       }

data/test/rbbt/ner/test_annotations.rb

@@ -0,0 +1,8 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
+
+class TestClass < Test::Unit::TestCase
+  def test_true
+    assert true
+  end
+end
+

data/test/rbbt/ner/test_banner.rb

@@ -4,11 +4,11 @@ require 'test/unit'
 
 class TestBanner < Test::Unit::TestCase
 
-  def test_extract
+  def test_match
     begin
       ner = Banner.new
 
-      mentions = ner.extract(" The P-ITIM-compelled multi-phosphoprotein complex binds to and activates SHP-2, which in turn dephosphorylates SHIP and Shc and probably other substrates.")
+      mentions = ner.match(" The P-ITIM-compelled multi-phosphoprotein complex binds to and activates SHP-2, which in turn dephosphorylates SHIP and Shc and probably other substrates.")
       ["SHP - 2", "SHIP", "Shc"].each{|mention| 
         assert(mentions.include? mention)
       }

data/test/rbbt/ner/test_oscar3.rb

@@ -6,12 +6,12 @@ require 'test/unit'
 class TestOSCAR3 < Test::Unit::TestCase
 
 
-  def test_extract
+  def test_match
     begin
       ner = OSCAR3.new
       str  = "Alternatively, rearrangement of O-(ω-haloalkyl)esters 34 of 2-carboethoxy-N-hydroxypyridine-2-selone affords azonianaphthalenium halides 37 in 79% yield"
 
-      mentions = ner.extract(str, "CM", false)
+      mentions = ner.match(str, "CM", false)
       good_mentions = ["2-carboethoxy-N-hydroxypyridine-2-selone", "O-(ω-haloalkyl)esters"]
 
       good_mentions.each{|mention| 
@@ -22,4 +22,37 @@ class TestOSCAR3 < Test::Unit::TestCase
       puts $!.backtrace
     end
   end
+
+  def test_ranges
+    begin
+      ner = OSCAR3.new
+      str =<<-EOF 
+This sentence talks about 2-carboethoxy-N-hydroxypyridine-2-selone.
+This sentence talks about 2-carboethoxy-N-hydroxypyridine-2-selone.
+This sentence talks about 2-carboethoxy-N-hydroxypyridine-2-selone.
+This sentence talks about 2-carboethoxy-N-hydroxypyridine-2-selone.
+This otherone talks about O-(ω-haloalkyl)esters.
+This otherone talks about O-(ω-haloalkyl)esters.
+This otherone talks about O-(ω-haloalkyl)esters.
+
+This otherone talks about O-(ω-haloalkyl)esters.
+This otherone talks about O-(ω-haloalkyl)esters.
+      EOF
+
+
+      mentions = ner.match(str, "CM", false)
+
+      str_original = str.dup
+      mentions.each do |mention|
+        str[mention.range] = mention
+      end
+
+      assert_equal str_original, str
+
+    rescue
+      puts $!.message
+      puts $!.backtrace
+    end
+  end
+
 end

data/test/rbbt/ner/test_regexpNER.rb

@@ -1,56 +1,104 @@
 require File.dirname(__FILE__) + '/../../test_helper'
-require 'rbbt-util'
 require 'rbbt/ner/regexpNER'
-require 'rbbt/sources/polysearch'
-require 'test/unit'
 
 class TestRegExpNER < Test::Unit::TestCase
-  def test_true
-    assert true
-  end
-  def _test_class
-    text = "a bc d e f g h i j k  l m n o p q one two"
+  def test_match_regexp
+    sentence = "In this sentence I should find this and 'that'"
 
-    lexicon =<<-EOF
-C1,a,x,xx,xxx
-C2,bc,y,yy,yyy
-C3,i,z,zz,zzz,m,one two
-    EOF
+    regexp = /this/
+    matches = RegExpNER.match_regexp(sentence, regexp)
 
-    file = TmpFile.tmp_file
-    File.open(file, 'w'){|f| f.write lexicon}
+    assert_equal ["this", "this"], matches
+    assert_equal "In ".length, matches[0].offset
+    assert_equal "In this sentence I should find ".length, matches[1].offset
 
-    r = RegExpNER.new(file, :sep => ',', :stopwords => false)
-    assert_equal(['a', 'bc', 'i', 'm','one two'].sort, r.match_hash(text).values.flatten.sort)
+    regexp_list = [/this/, /that/]
+    matches = RegExpNER.match_regexp_list(sentence, regexp_list)
 
-    r = RegExpNER.new(file, :sep => ',', :stopwords => true)
-    assert_equal(['bc', 'm','one two'].sort,r.match_hash(text).values.flatten.sort)
+    assert_equal ["this", "this", "that"], matches
+    assert_equal "In ".length, matches[0].offset
+    assert_equal "In this sentence I should find ".length, matches[1].offset
 
+    regexp_hash = {:this => /this/, :that => /that/}
+    matches = RegExpNER.match_regexp_hash(sentence, regexp_hash)
 
-    FileUtils.rm file
+    assert_equal ["this", "this", "that"].sort, matches.sort
+    assert_equal "In ".length, matches.select{|m| m.type == :this}[0].offset
+    assert_equal "In this sentence I should find ".length, matches.select{|m| m.type == :this}[1].offset
+    assert_equal :this, matches.select{|m| m.type == :this}[0].type
   end
 
-  def _test_persistence
-    text = "a bc d e f g h i j k  l m n o p q one two"
+  def test_define_regexps
+    sentence = "In this sentence I should find this and 'that'"
 
-    lexicon =<<-EOF
-C1,a,x,xx,xxx
-C2,bc,y,yy,yyy
-C3,i,z,zz,zzz,m,one two
-    EOF
+    ner = RegExpNER.new
+    ner.define_regexp do
+      this /this/
+      that /that/
+    end
+
+    matches = ner.entities(sentence)
+    assert_equal ["this", "this", "that"].sort, matches.sort
+    assert_equal "In ".length, matches.select{|m| m.type == :this }[0].offset
+    assert_equal "In this sentence I should find ".length, matches.select{|m| m.type == :this }[1].offset
+    assert_equal :this, matches.select{|m| m.type == :this }[0].type
+  end
 
-    file = TmpFile.tmp_file
-    File.open(file, 'w'){|f| f.write lexicon}
 
-    r = RegExpNER.new(file, :sep => ',', :stopwords => false, :persistence => true)
-    assert_equal(['a', 'bc', 'i', 'm','one two'].sort, r.match_hash(text).values.flatten.sort)
+  def test_entities
+    sentence = "In this sentence I should find this and 'that'"
 
-    r = RegExpNER.new(file, :sep => ',', :stopwords => true, :persistence => true)
-    assert_equal(['bc', 'm','one two'].sort,r.match_hash(text).values.flatten.sort)
+    ner = RegExpNER.new({:this => /this/, :that => /that/})
+    matches = ner.entities(sentence)
+    assert_equal ["this", "this", "that"].sort, matches.sort
+    assert_equal "In ".length, matches.select{|m| m.type == :this}[0].offset
+    assert_equal "In this sentence I should find ".length, matches.select{|m| m.type == :this}[1].offset
+    assert_equal :this, matches.select{|m| m.type == :this}[0].type
 
+    Annotated.annotate(sentence)
+    ner_this = RegExpNER.new({:this => /this/})
+    ner_that = RegExpNER.new({:that => /that/})
+    sentence.annotations += ner_this.entities(sentence)
+    sentence.annotations += ner_that.entities(sentence)
+    matches = sentence.annotations
 
-    FileUtils.rm file
+    assert_equal ["this", "this", "that"].sort, matches.sort
+    assert_equal "In ".length, matches.select{|m| m.type == :this}[0].offset
+    assert_equal "In this sentence I should find ".length, matches.select{|m| m.type == :this}[1].offset
+    assert_equal :this, matches.select{|m| m.type == :this}[0].type
   end
-end
 
+  def test_entities_captures
+    sentence = "In this sentence I should find this and 'that'"
+
+    ner = RegExpNER.new({:this => /this/, :that => /that/, :should => /I (should)/})
+    matches = ner.entities(sentence)
+    assert_equal ["this", "this", "that", "should"].sort, matches.sort
+    assert_equal "In this sentence I ".length, matches.select{|m| m.type == :should}[0].offset
+    assert_equal :should, matches.select{|m| m.type == :should}[0].type
+  end
 
+  def test_regexp_order
+    text =<<-EOF
+  * Human AUC 0-24h= 7591 ng.h/ml at 30 mg/day    In mice, dietary administration of aripiprazole at doses of 1, 3, and 10 asdf mg/kg/day for 104 weeks was
+  associated with increased incidences of mammary tumors, namely adenocarcinomas
+    EOF
+
+
+
+    regexp = RegExpNER.new
+    regexp.define_regexp do
+      dosage           /\d+\s*(?:[mnukg]{1,2}|mol)(?:\/[mnguk]{1,2})?(?:\/day|d|hour|h|minute|min|m)?/i
+      time             /[\d\.]+\s+(?:minute|hour|day|week|mounth|year)s?/i
+    end
+
+    offsets = {
+      "7591 ng" => 21,
+      "30 mg/day" => 37,
+      "104 weeks" => 142,
+    }
+    regexp.match(text).each do |entity|
+      assert_equal offsets[entity], entity.offset
+    end
+  end
+end

data/test/rbbt/ner/test_token_trieNER.rb

@@ -0,0 +1,112 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
+require 'rbbt/ner/token_trieNER'
+require 'rbbt/util/tmpfile'
+
+class TestTokenTrieNER < Test::Unit::TestCase
+
+  def test_tokenize
+    assert_equal ['a' , 'b', ',', 'c'], TokenTrieNER.tokenize('a b, c')
+
+    assert_equal 10, TokenTrieNER.tokenize('123456789 12345').last.offset
+    assert_equal 0, TokenTrieNER.tokenize('123456789 12345').first.offset
+
+
+    text = '123456789 12345'
+    assert_equal '12345', text[TokenTrieNER.tokenize('123456789 12345').last.range]
+  end
+
+  def test_merge
+    tokens = %w(a b c)
+    index = {'a' => {'b' => {'c' => {:END => [TokenTrieNER::Code.new 'CODE']}}}}
+
+    assert_equal 'CODE', TokenTrieNER.merge({}, TokenTrieNER.index_for_tokens(tokens, 'CODE'))['a']['b']['c'][:END].first.value
+  end
+
+  def test_process
+    lexicon =<<-EOF
+C1;aa;AA;bb b
+C2;11;22;3 3;bb
+    EOF
+
+    TmpFile.with_file(lexicon) do |file|
+
+      index = TokenTrieNER.process(TSV.new(file, :sep => ';', :flatten => true))
+
+      assert_equal ['AA', 'aa', 'bb', '11', '22', '3'].sort, index.keys.sort
+      assert_equal [:END], index['aa'].keys
+      assert index['bb'].keys.include? 'b'
+      assert index['bb'].keys.include? :END
+    end
+  end
+
+  def test_find
+    lexicon =<<-EOF
+C1;aa;AA;bb b
+C2;11;22;3 3;bb
+    EOF
+
+
+    TmpFile.with_file(lexicon) do |file|
+      index = TokenTrieNER.process(TSV.new(file, :sep => ';', :flatten => true))
+
+      assert TokenTrieNER.find(index, TokenTrieNER.tokenize('aa asdf'), false).first.collect{|c| c.value}.include?   'C1'
+      assert_equal %w(aa), TokenTrieNER.find(index, TokenTrieNER.tokenize('aa asdf'), false).last
+
+      assert TokenTrieNER.find(index, TokenTrieNER.tokenize('aa asdf'), true).first.collect{|c| c.value}.include?    'C1'
+
+      assert TokenTrieNER.find(index, TokenTrieNER.tokenize('bb b asdf'), true).first.collect{|c| c.value}.include?  'C1'
+      assert_equal %w(bb b), TokenTrieNER.find(index, TokenTrieNER.tokenize('bb b asdf'), true).last
+
+      assert TokenTrieNER.find(index, TokenTrieNER.tokenize('bb b asdf'), false).first.collect{|c| c.value}.include? 'C2'
+      assert_equal %w(bb), TokenTrieNER.find(index, TokenTrieNER.tokenize('bb b asdf'), false).last
+
+      assert TokenTrieNER.find(index, TokenTrieNER.tokenize('bb asdf'), false).first.collect{|c| c.value}.include?   'C2'
+    end
+  end
+
+  def test_match
+    lexicon =<<-EOF
+C1;aa;AA;bb b
+C2;11;22;3 3;bb
+    EOF
+
+    TmpFile.with_file(lexicon) do |file|
+      index = TokenTrieNER.new(file, nil, :sep => ';')
+
+      assert index.match(' asdfa dsf asdf aa asdfasdf ').select{|m| m.code.include? 'C1'}.any?
+    end
+  end
+
+  def _test_polysearch_long_match
+    begin
+      require 'rbbt/sources/polysearch'
+    rescue
+      puts "Polysearch is not available. Some test have not ran."
+      assert true
+      return
+    end
+
+    sentence = "mammary and pituitary neoplasms as well as other drug-related mammary/reproductive tissue alterations in females were considered"
+
+    index = TokenTrieNER.new Rbbt.find_datafile('organ')
+    assert index.match(sentence).collect{|m| m.code}.flatten.include? 'OR00063'
+
+    index = TokenTrieNER.new Rbbt.find_datafile('disease')
+    assert index.match(sentence).collect{|m| m.code}.flatten.include? 'DID44386'
+
+    index = TokenTrieNER.new Rbbt.find_datafile('disease'), Rbbt.find_datafile('organ')
+    assert index.match(sentence).collect{|m| m.code}.flatten.include? 'DID44386'
+
+    index = TokenTrieNER.new Rbbt.find_datafile('disease'), Rbbt.find_datafile('organ')
+    assert index.match(sentence).collect{|m| m.code}.flatten.include? 'DID44386'
+
+    index = TokenTrieNER.new Rbbt.find_datafile('organ')
+    assert index.match(sentence).collect{|m| m.code}.flatten.include? 'OR00063'
+    index.merge Rbbt.find_datafile('disease')
+    assert ! index.match(sentence).collect{|m| m.code}.flatten.include?('OR00063')
+    assert index.match(sentence).collect{|m| m.code}.flatten.include? 'DID44386'
+  end
+
+
+end
+