rbbt-text 0.2.1 → 0.5.0

data/test/rbbt/ner/test_regexpNER.rb

@@ -1,4 +1,4 @@
-require File.dirname(__FILE__) + '/../../test_helper'
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../test_helper.rb')
 require 'rbbt/ner/regexpNER'
 
 class TestRegExpNER < Test::Unit::TestCase

data/test/rbbt/ner/test_rnorm.rb

@@ -0,0 +1,47 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../test_helper.rb')
+require 'rbbt/ner/rnorm'
+require 'rbbt/util/open'
+require 'rbbt/util/tmpfile'
+require 'test/unit' 
+ 
+class TestRNORM < Test::Unit::TestCase
+ 
+  def setup 
+    tmp = TmpFile.tmp_file("test-rnorm-")
+    lexicon =<<-EOT
+S000000029	YAL031C	GIP4	FUN21
+S000000030	YAL032C	PRP45	FUN20
+S000000031	YAL033W	POP5	FUN53 
+S000000374	YBR170C	NPL4	HRD4
+S000000375	GENE1	BBB	CCC
+S000000376	AAA	GENE1	DDD
+	EOT
+
+    Open.write(tmp, lexicon)
+
+    @norm = Normalizer.new(tmp) 
+    FileUtils.rm tmp
+  end
+
+  def test_match
+     assert_equal(["S000000029"], @norm.match("FUN21"))
+     assert_equal(["S000000030", "S000000029", "S000000031"], @norm.match("FUN"))
+     assert_equal(["S000000030", "S000000029", "S000000031"], @norm.match("FUN 2"))
+     assert_equal(["S000000030", "S000000029", "S000000031"], @norm.match("FUN 21")) 
+     assert_equal([], @norm.match("GER4"))
+
+     @norm.match("FUN21")
+  end 
+
+  def test_select
+    assert_equal(["S000000029"], @norm.select(["S000000030", "S000000029", "S000000031"],"FUN 21"))
+  end
+
+  def test_resolve
+    assert_equal(["S000000029"], @norm.resolve("FUN 21"))
+  end
+
+  def test_order
+    assert_equal(["S000000375"], @norm.resolve("GENE1"))
+  end
+end

data/test/rbbt/ner/test_token_trieNER.rb

@@ -10,16 +10,16 @@ class TestTokenTrieNER < Test::Unit::TestCase
     assert_equal 10, TokenTrieNER.tokenize('123456789 12345').last.offset
     assert_equal 0, TokenTrieNER.tokenize('123456789 12345').first.offset
 
-
     text = '123456789 12345'
     assert_equal '12345', text[TokenTrieNER.tokenize('123456789 12345').last.range]
   end
 
   def test_merge
     tokens = %w(a b c)
-    index = {'a' => {'b' => {'c' => {:END => [TokenTrieNER::Code.new 'CODE']}}}}
+    tokens.extend TokenTrieNER::EnumeratedArray
+    index = {'a' => {'b' => {'c' => {:END => [TokenTrieNER::Code.new('CODE')]}}}}
 
-    assert_equal 'CODE', TokenTrieNER.merge({}, TokenTrieNER.index_for_tokens(tokens, 'CODE'))['a']['b']['c'][:END].first.value
+    assert_equal 'CODE', TokenTrieNER.merge({}, TokenTrieNER.index_for_tokens(tokens, 'CODE'))['a']['b']['c'][:END].first.code
   end
 
   def test_process
@@ -30,7 +30,7 @@ C2;11;22;3 3;bb
 
     TmpFile.with_file(lexicon) do |file|
 
-      index = TokenTrieNER.process(TSV.new(file, :sep => ';', :flatten => true))
+      index = TokenTrieNER.process({}, TSV.new(file, :flat, :sep => ';'))
 
       assert_equal ['AA', 'aa', 'bb', '11', '22', '3'].sort, index.keys.sort
       assert_equal [:END], index['aa'].keys
@@ -47,20 +47,20 @@ C2;11;22;3 3;bb
 
 
     TmpFile.with_file(lexicon) do |file|
-      index = TokenTrieNER.process(TSV.new(file, :sep => ';', :flatten => true))
+      index = TokenTrieNER.process({}, TSV.new(file, :sep => ';', :type => :flat ))
 
-      assert TokenTrieNER.find(index, TokenTrieNER.tokenize('aa asdf'), false).first.collect{|c| c.value}.include?   'C1'
-      assert_equal %w(aa), TokenTrieNER.find(index, TokenTrieNER.tokenize('aa asdf'), false).last
+      assert TokenTrieNER.find(index, TokenTrieNER.tokenize('aa asdf').extend(TokenTrieNER::EnumeratedArray), false).first.collect{|c| c.code}.include?   'C1'
+      assert_equal %w(aa), TokenTrieNER.find(index, TokenTrieNER.tokenize('aa asdf').extend(TokenTrieNER::EnumeratedArray), false).last
 
-      assert TokenTrieNER.find(index, TokenTrieNER.tokenize('aa asdf'), true).first.collect{|c| c.value}.include?    'C1'
+      assert TokenTrieNER.find(index, TokenTrieNER.tokenize('aa asdf').extend(TokenTrieNER::EnumeratedArray), true).first.collect{|c| c.code}.include?    'C1'
 
-      assert TokenTrieNER.find(index, TokenTrieNER.tokenize('bb b asdf'), true).first.collect{|c| c.value}.include?  'C1'
-      assert_equal %w(bb b), TokenTrieNER.find(index, TokenTrieNER.tokenize('bb b asdf'), true).last
+      assert TokenTrieNER.find(index, TokenTrieNER.tokenize('bb b asdf').extend(TokenTrieNER::EnumeratedArray), true).first.collect{|c| c.code}.include?  'C1'
+      assert_equal %w(bb b), TokenTrieNER.find(index, TokenTrieNER.tokenize('bb b asdf').extend(TokenTrieNER::EnumeratedArray), true).last
 
-      assert TokenTrieNER.find(index, TokenTrieNER.tokenize('bb b asdf'), false).first.collect{|c| c.value}.include? 'C2'
-      assert_equal %w(bb), TokenTrieNER.find(index, TokenTrieNER.tokenize('bb b asdf'), false).last
+      assert TokenTrieNER.find(index, TokenTrieNER.tokenize('bb b asdf').extend(TokenTrieNER::EnumeratedArray), false).first.collect{|c| c.code}.include? 'C2'
+      assert_equal %w(bb), TokenTrieNER.find(index, TokenTrieNER.tokenize('bb b asdf').extend(TokenTrieNER::EnumeratedArray), false).last
 
-      assert TokenTrieNER.find(index, TokenTrieNER.tokenize('bb asdf'), false).first.collect{|c| c.value}.include?   'C2'
+      assert TokenTrieNER.find(index, TokenTrieNER.tokenize('bb asdf').extend(TokenTrieNER::EnumeratedArray), false).first.collect{|c| c.code}.include? 'C2'
     end
   end
 
@@ -71,42 +71,67 @@ C2;11;22;3 3;bb
     EOF
 
     TmpFile.with_file(lexicon) do |file|
-      index = TokenTrieNER.new(file, nil, :sep => ';')
+      index = TokenTrieNER.new("test", TSV.new(file, :flat, :sep => ';'))
 
+      index.match(' asdfa dsf asdf aa asdfasdf ')
       assert index.match(' asdfa dsf asdf aa asdfasdf ').select{|m| m.code.include? 'C1'}.any?
     end
   end
 
-  def _test_polysearch_long_match
-    begin
-      require 'rbbt/sources/polysearch'
-    rescue
-      puts "Polysearch is not available. Some test have not ran."
-      assert true
-      return
+  def test_slack
+    lexicon =<<-EOF
+C1;aa;AA;bb cc cc b
+C2;11;22;3 3;bb
+    EOF
+
+    TmpFile.with_file(lexicon) do |file|
+      index = TokenTrieNER.new({})
+      index.slack = Proc.new{|t| t =~ /^c*$/}
+
+      index.merge TSV.new(file, :flat, :sep => ';')
+
+      assert index.match(' aaaaa 3 cc 3').select{|m| m.code.include? 'C2'}.any?
+      assert index.match(' bb cc b').select{|m| m.code.include? 'C1'}.any?
+      assert index.match(' bb b').select{|m| m.code.include? 'C1'}.any?
     end
+  end
 
-    sentence = "mammary and pituitary neoplasms as well as other drug-related mammary/reproductive tissue alterations in females were considered"
+  def test_own_tokens
+    lexicon =<<-EOF
+C1;aa;AA;bb cc cc b
+C2;11;22;3 3;bb
+    EOF
 
-    index = TokenTrieNER.new Rbbt.find_datafile('organ')
-    assert index.match(sentence).collect{|m| m.code}.flatten.include? 'OR00063'
+    TmpFile.with_file(lexicon) do |file|
+      index = TokenTrieNER.new({})
+      index.slack = Proc.new{|t| t =~ /^c*$/}
 
-    index = TokenTrieNER.new Rbbt.find_datafile('disease')
-    assert index.match(sentence).collect{|m| m.code}.flatten.include? 'DID44386'
+      index.merge TSV.new(file, :flat, :sep => ';')
 
-    index = TokenTrieNER.new Rbbt.find_datafile('disease'), Rbbt.find_datafile('organ')
-    assert index.match(sentence).collect{|m| m.code}.flatten.include? 'DID44386'
+      assert index.match(Token.tokenize('3 cc 3')).select{|m| m.code.include? 'C2'}.any?
+    end
+  end
 
-    index = TokenTrieNER.new Rbbt.find_datafile('disease'), Rbbt.find_datafile('organ')
-    assert index.match(sentence).collect{|m| m.code}.flatten.include? 'DID44386'
+  def test_proc_index
+    index = TokenTrieNER.new({})
+    index.merge({ "aa" => {:PROCS => {Proc.new{|c| c == 'c'} => {:END  => [TokenTrieNER::Code.new(:entity, :C1)]}}}})
 
-    index = TokenTrieNER.new Rbbt.find_datafile('organ')
-    assert index.match(sentence).collect{|m| m.code}.flatten.include? 'OR00063'
-    index.merge Rbbt.find_datafile('disease')
-    assert ! index.match(sentence).collect{|m| m.code}.flatten.include?('OR00063')
-    assert index.match(sentence).collect{|m| m.code}.flatten.include? 'DID44386'
+    assert index.match(Token.tokenize('3 cc 3 aa c ddd')).select{|m| m.code.include? :entity}.any?
   end
 
+  def test_persistence
+    lexicon =<<-EOF
+C1;aa;AA;bb b
+C2;11;22;3 3;bb
+    EOF
+
+    TmpFile.with_file(lexicon) do |file|
+      index = TokenTrieNER.new("test", TSV.new(file, :flat, :sep => ';'), :persistence => true)
+
+      index.match(' asdfa dsf asdf aa asdfasdf ')
+      assert index.match(' asdfa dsf asdf aa asdfasdf ').select{|m| m.code.include? 'C1'}.any?
+    end
+  end
 
 end
 

data/test/rbbt/nlp/test_nlp.rb

@@ -0,0 +1,88 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
+require 'rbbt/nlp/nlp'
+
+text=<<-EOF
+Atypical teratoid/rhabdoid tumors (AT/RTs) are highly aggressive brain tumors
+of early childhood poorly responding to therapy. The majority of cases show
+inactivation of SMARCB1 (INI1, hSNF5, BAF47), a core member of the adenosine
+triphosphate (ATP)-dependent SWI/SNF chromatin-remodeling complex. We here
+report the case of a supratentorial AT/RT in a 9-month-old boy, which showed
+retained SMARCB1 staining on immunohistochemistry and lacked genetic
+alterations of SMARCB1. Instead, the tumor showed loss of protein expression of
+another SWI/SNF chromatin-remodeling complex member, the ATPase subunit SMARCA4
+(BRG1) due to a homozygous SMARCA4 mutation [c.2032C>T (p.Q678X)].  Our
+findings highlight the role of SMARCA4 in the pathogenesis of SMARCB1-positive
+AT/RT and the usefulness of antibodies directed against SMARCA4 in this
+diagnostic setting.
+  EOF
+
+class TestClass < Test::Unit::TestCase
+
+  def test_sentences
+    text =<<-EOF
+This is a 
+sentence. This is
+another sentence. 
+    EOF
+
+    assert_equal 2, NLP.geniass_sentence_splitter(text).length
+    assert_equal "This is a \nsentence. ", NLP.geniass_sentence_splitter(text).first
+  end
+
+  def test_gdep_parse_sentences
+    text =<<-EOF
+Atypical teratoid/rhabdoid tumors (AT/RTs)
+are highly aggressive brain
+tumors of early childhood poorly 
+responding to therapy.
+    EOF
+
+    NLP.gdep_parse_sentences_extension([text, text]).zip([text,text]).each do |segment_list, sentence|
+      segment_list.each do |segment|
+        assert_equal sentence[segment.range], segment
+      end
+    end
+  end
+
+  def test_gdep_chunks
+    text =<<-EOF
+Atypical teratoid/rhabdoid tumors (AT/RTs)
+are highly aggressive brain
+tumors of early childhood poorly 
+responding to therapy.
+    EOF
+
+    NLP.gdep_parse_sentences_extension([text, text]).zip([text,text]).each do |segment_list, sentence|
+      chunk_list = NLP.gdep_chunks(sentence, segment_list)
+      chunk_list.each do |segment|
+        assert_equal sentence[segment.range], segment
+      end
+
+      assert chunk_list.select{|c| c =~ /rhabdoid/}.first.parts.include? "tumors"
+    end
+ 
+  end
+
+  def test_merge_chunks
+    text =<<-EOF
+Atypical teratoid/rhabdoid tumors (AT/RTs)
+where found to be like highly aggressive brain
+tumors of early childhood poorly 
+responding to therapy.
+    EOF
+
+    NLP.gdep_parse_sentences_extension([text, text]).zip([text,text]).each do |segment_list, sentence|
+      chunk_list = NLP.gdep_chunks(sentence, segment_list)
+      new_chunk_list = NLP.merge_vp_chunks(chunk_list)
+      
+      new_chunk_list.each do |segment|
+        assert_equal sentence[segment.range], segment
+      end
+
+      assert new_chunk_list.select{|c| c.type == "VP"}.first.parts.include? "found"
+      assert new_chunk_list.select{|c| c.type == "VP"}.first.parts.include? "to"
+      assert new_chunk_list.select{|c| c.type == "VP"}.first.parts.include? "be"
+    end
+  end
+end
+ 

data/test/test_helper.rb

@@ -2,8 +2,28 @@ require 'test/unit'
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 
+require 'rbbt'
+require 'rbbt/util/persistence'
+require 'rbbt/util/tmpfile'
+require 'rbbt/util/log'
+require 'rbbt/corpus/document_repo'
+
 class Test::Unit::TestCase
   def test_datafile(file)
     File.join(File.dirname(__FILE__), 'data', file)
   end
+
+  def setup
+    FileUtils.mkdir_p Rbbt.tmp.test.persistence.find(:user)
+    Persistence.cachedir = Rbbt.tmp.test.persistence.find :user
+  end
+
+  def teardown
+    FileUtils.rm_rf Rbbt.tmp.test.find :user
+    TCHash::CONNECTIONS.values.each do |c| c.close end
+    TCHash::CONNECTIONS.clear
+    DocumentRepo::CONNECTIONS.values.each do |c| c.close end
+    DocumentRepo::CONNECTIONS.clear
+  end
+
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: rbbt-text
 version: !ruby/object:Gem::Version 
-  hash: 21
+  hash: 11
   prerelease: 
   segments: 
   - 0
-  - 2
-  - 1
-  version: 0.2.1
+  - 5
+  - 0
+  version: 0.5.0
 platform: ruby
 authors: 
 - Miguel Vazquez
@@ -15,8 +15,8 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-01-30 00:00:00 +01:00
-default_executable: 
+date: 2011-07-05 00:00:00 +02:00
+default_executable: get_ppis.rb
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: rbbt-util
@@ -46,10 +46,38 @@ dependencies:
         version: "0"
   type: :runtime
   version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: libxml-ruby
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: json
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id004
 description: "Text mining tools: named entity recognition and normalization, document classification, bag-of-words, dictionaries, etc"
 email: miguel.vazquez@fdi.ucm.es
-executables: []
-
+executables: 
+- get_ppis.rb
 extensions: []
 
 extra_rdoc_files: []
@@ -58,28 +86,64 @@ files:
 - lib/rbbt/bow/bow.rb
 - lib/rbbt/bow/dictionary.rb
 - lib/rbbt/bow/misc.rb
+- lib/rbbt/corpus/corpus.rb
+- lib/rbbt/corpus/document.rb
+- lib/rbbt/corpus/document_repo.rb
+- lib/rbbt/corpus/sources/pubmed.rb
 - lib/rbbt/ner/NER.rb
 - lib/rbbt/ner/abner.rb
 - lib/rbbt/ner/annotations.rb
+- lib/rbbt/ner/annotations/annotated.rb
+- lib/rbbt/ner/annotations/named_entity.rb
+- lib/rbbt/ner/annotations/relations.rb
+- lib/rbbt/ner/annotations/token.rb
+- lib/rbbt/ner/annotations/transformed.rb
 - lib/rbbt/ner/banner.rb
+- lib/rbbt/ner/chemical_tagger.rb
+- lib/rbbt/ner/ngram_prefix_dictionary.rb
 - lib/rbbt/ner/oscar3.rb
+- lib/rbbt/ner/oscar4.rb
+- lib/rbbt/ner/patterns.rb
 - lib/rbbt/ner/regexpNER.rb
+- lib/rbbt/ner/rnorm.rb
+- lib/rbbt/ner/rnorm/cue_index.rb
+- lib/rbbt/ner/rnorm/tokens.rb
 - lib/rbbt/ner/token_trieNER.rb
+- lib/rbbt/nlp/genia/sentence_splitter.rb
+- lib/rbbt/nlp/nlp.rb
 - share/install/software/ABNER
 - share/install/software/BANNER
+- share/install/software/ChemicalTagger
+- share/install/software/Gdep
+- share/install/software/Geniass
 - share/install/software/OSCAR3
-- share/stopwords
+- share/install/software/OSCAR4
+- share/install/software/StanfordParser
+- share/patterns/drug_induce_disease
+- share/rnorm/cue_default
+- share/rnorm/tokens_default
+- share/wordlists/stopwords
+- test/test_helper.rb
 - test/rbbt/bow/test_bow.rb
 - test/rbbt/bow/test_dictionary.rb
 - test/rbbt/bow/test_misc.rb
-- test/rbbt/ner/test_NER.rb
+- test/rbbt/ner/test_regexpNER.rb
 - test/rbbt/ner/test_abner.rb
-- test/rbbt/ner/test_annotations.rb
 - test/rbbt/ner/test_banner.rb
-- test/rbbt/ner/test_oscar3.rb
-- test/rbbt/ner/test_regexpNER.rb
+- test/rbbt/ner/annotations/test_transformed.rb
+- test/rbbt/ner/annotations/test_named_entity.rb
 - test/rbbt/ner/test_token_trieNER.rb
-- test/test_helper.rb
+- test/rbbt/ner/test_annotations.rb
+- test/rbbt/ner/test_patterns.rb
+- test/rbbt/ner/test_NER.rb
+- test/rbbt/ner/test_rnorm.rb
+- test/rbbt/ner/test_oscar4.rb
+- test/rbbt/ner/test_chemical_tagger.rb
+- test/rbbt/ner/test_ngram_prefix_dictionary.rb
+- test/rbbt/nlp/test_nlp.rb
+- test/rbbt/corpus/test_corpus.rb
+- test/rbbt/corpus/test_document.rb
+- bin/get_ppis.rb
 has_rdoc: true
 homepage: http://github.com/mikisvaz/rbbt-util
 licenses: []
@@ -110,19 +174,28 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.4.2
+rubygems_version: 1.6.2
 signing_key: 
 specification_version: 3
 summary: Text mining tools for the Ruby Bioinformatics Toolkit (rbbt)
 test_files: 
+- test/test_helper.rb
 - test/rbbt/bow/test_bow.rb
 - test/rbbt/bow/test_dictionary.rb
 - test/rbbt/bow/test_misc.rb
-- test/rbbt/ner/test_NER.rb
+- test/rbbt/ner/test_regexpNER.rb
 - test/rbbt/ner/test_abner.rb
-- test/rbbt/ner/test_annotations.rb
 - test/rbbt/ner/test_banner.rb
-- test/rbbt/ner/test_oscar3.rb
-- test/rbbt/ner/test_regexpNER.rb
+- test/rbbt/ner/annotations/test_transformed.rb
+- test/rbbt/ner/annotations/test_named_entity.rb
 - test/rbbt/ner/test_token_trieNER.rb
-- test/test_helper.rb
+- test/rbbt/ner/test_annotations.rb
+- test/rbbt/ner/test_patterns.rb
+- test/rbbt/ner/test_NER.rb
+- test/rbbt/ner/test_rnorm.rb
+- test/rbbt/ner/test_oscar4.rb
+- test/rbbt/ner/test_chemical_tagger.rb
+- test/rbbt/ner/test_ngram_prefix_dictionary.rb
+- test/rbbt/nlp/test_nlp.rb
+- test/rbbt/corpus/test_corpus.rb
+- test/rbbt/corpus/test_document.rb