rbbt-text 0.2.0 → 0.2.1

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: rbbt-text
 version: !ruby/object:Gem::Version 
-  hash: 23
-  prerelease: false
+  hash: 21
+  prerelease: 
   segments: 
   - 0
   - 2
-  - 0
-  version: 0.2.0
+  - 1
+  version: 0.2.1
 platform: ruby
 authors: 
 - Miguel Vazquez
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-12-22 00:00:00 +01:00
+date: 2011-01-30 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -58,12 +58,13 @@ files:
 - lib/rbbt/bow/bow.rb
 - lib/rbbt/bow/dictionary.rb
 - lib/rbbt/bow/misc.rb
+- lib/rbbt/ner/NER.rb
 - lib/rbbt/ner/abner.rb
+- lib/rbbt/ner/annotations.rb
 - lib/rbbt/ner/banner.rb
-- lib/rbbt/ner/named_entity.rb
 - lib/rbbt/ner/oscar3.rb
 - lib/rbbt/ner/regexpNER.rb
-- lib/rbbt/ner/tokenNER.rb
+- lib/rbbt/ner/token_trieNER.rb
 - share/install/software/ABNER
 - share/install/software/BANNER
 - share/install/software/OSCAR3
@@ -71,12 +72,13 @@ files:
 - test/rbbt/bow/test_bow.rb
 - test/rbbt/bow/test_dictionary.rb
 - test/rbbt/bow/test_misc.rb
+- test/rbbt/ner/test_NER.rb
 - test/rbbt/ner/test_abner.rb
+- test/rbbt/ner/test_annotations.rb
 - test/rbbt/ner/test_banner.rb
-- test/rbbt/ner/test_named_entity.rb
 - test/rbbt/ner/test_oscar3.rb
 - test/rbbt/ner/test_regexpNER.rb
-- test/rbbt/ner/test_tokenNER.rb
+- test/rbbt/ner/test_token_trieNER.rb
 - test/test_helper.rb
 has_rdoc: true
 homepage: http://github.com/mikisvaz/rbbt-util
@@ -108,7 +110,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.7
+rubygems_version: 1.4.2
 signing_key: 
 specification_version: 3
 summary: Text mining tools for the Ruby Bioinformatics Toolkit (rbbt)
@@ -116,10 +118,11 @@ test_files:
 - test/rbbt/bow/test_bow.rb
 - test/rbbt/bow/test_dictionary.rb
 - test/rbbt/bow/test_misc.rb
+- test/rbbt/ner/test_NER.rb
 - test/rbbt/ner/test_abner.rb
+- test/rbbt/ner/test_annotations.rb
 - test/rbbt/ner/test_banner.rb
-- test/rbbt/ner/test_named_entity.rb
 - test/rbbt/ner/test_oscar3.rb
 - test/rbbt/ner/test_regexpNER.rb
-- test/rbbt/ner/test_tokenNER.rb
+- test/rbbt/ner/test_token_trieNER.rb
 - test/test_helper.rb

data/lib/rbbt/ner/named_entity.rb

@@ -1,11 +0,0 @@
-
-module NamedEntity
-  def self.annotate(string, type = nil, score = nil, range = nil)
-    string.extend NamedEntity
-    string.type = type
-    string.score = score
-    string.range = range
-  end
-
-  attr_accessor :type, :score, :range
-end

data/lib/rbbt/ner/tokenNER.rb

@@ -1,237 +0,0 @@
-require 'rbbt-util'
-require 'rbbt/util/tsv'
-require 'rbbt/util/simpleDSL'
-require 'rbbt/ner/named_entity'
-
-class TokenNER
-  include SimpleDSL
-
-  module AnnotatedToken 
-    attr_accessor :original, :range
-  end
-
-  def self.clean(token)
-    if token.length > 3
-      token.downcase
-    else
-      token
-    end
-  end
-
-  def self.prepare_token(token, start)
-    clean_token = clean token
-    clean_token.extend AnnotatedToken
-    clean_token.original = token
-    clean_token.range = (start..(start + token.length - 1))
-    clean_token
-  end
-
-  def self.tokenize(text, split_at = /\s|(\(|\)|[-."':,])/, start = 0)
-
-    tokens = []
-    while matchdata = text.match(split_at)
-      tokens << prepare_token(matchdata.pre_match, start) unless matchdata.pre_match.empty?
-      tokens << prepare_token(matchdata.captures.first, start + matchdata.begin(1)) if matchdata.captures.any? and not matchdata.captures.first.empty?
-      start += matchdata.end(0)
-      text = matchdata.post_match
-    end
-    tokens << prepare_token(text, start) unless text.empty?
-
-    tokens
-  end
-
-  def self.match_regexp(text, regexp, start = 0)
-    chunks = []
-    matches = []
-    while matchdata = text.match(regexp)
-      pre   = matchdata.pre_match
-      post  = matchdata.post_match
-      match = matchdata[0]
-
-      if matchdata.captures.any?
-        more_pre, more_post = match.split(/#{matchdata.captures.first}/)
-        match = matchdata.captures.first
-        pre << more_pre
-        post = more_post << post
-      end
-
-      chunks << [pre, start]
-
-      matches << prepare_token(match, start + pre.length) unless match.empty?
-      start += pre.length + match.length
-      text = matchdata.post_match
-    end
-    chunks << [text, start]
-
-    [matches, chunks]
-  end
-
-  def self.match_regexps(text, regexps)
-    start = 0
-    chunks = [[text, 0]]
-
-    matches = []
-    regexps.each do |regexp, type|
-      
-      new_regexp_chunks = []
-      chunks.each do |chunk, start|
-        new_matches, new_chunk_chunks = match_regexp(chunk, regexp, start)
-
-        new_matches.each do |new_match|
-          new_match.extend NamedEntity
-          new_match.type = type
-          matches << new_match
-        end
-
-        new_regexp_chunks.concat new_chunk_chunks
-      end
-      chunks = new_regexp_chunks
-
-    end
-    [matches, chunks]
-  end
-
-  def self.tokenize_with_regexps(text, regexps = [], split_at = /\s|(\(|\)|[-."':,])/)
-    matches, chunks = match_regexps(text, regexps)
-
-    tokens = matches
-    chunks.each do |chunk, start|
-      tokens.concat tokenize(chunk, split_at, start)
-    end
-
-    tokens
-  end
-
-  def self.index_for_tokens(tokens, code)
-    if tokens.empty?
-      {:END => [code]}
-    else
-      {tokens.shift => index_for_tokens(tokens, code)}
-    end
-  end
-  
-  def self.merge(index1, index2)
-    index2.each do |key, new_index2|
-      case
-      when key == :END
-        index1[:END] ||= []
-        index1[:END] += new_index2
-        index1[:END].uniq!
-      when index1.include?(key)
-        merge(index1[key], new_index2)
-      else
-        index1[key] = new_index2
-      end
-    end
-  end
-
-  def self.process(hash)
-    index = {}
-    hash.each do |code, names|
-      names.each do |name|
-        next if name.empty? or name.length < 2
-        tokens = tokenize name
-
-        merge(index, index_for_tokens(tokens, code)) unless tokens.empty?
-      end
-    end
-    index
-  end
-
-  attr_accessor :index, :longest_match
-  def initialize(file, options = {})
-    options = Misc.add_defaults options, :flatten => true, :longest_match => true
-    @longest_match = options.delete :longest_match
-
-    @regexps = options[:regexps] || []
-
-    file = [file] unless Array === file
-    @index = {}
-    file.each do |f| TokenNER.merge(@index, TokenNER.process(TSV.new(f, options))) end
-  end
-
-  def merge(new)
-    case
-    when TokenNER === new
-      TokenNER.merge(@index, new.index)
-    when Hash === new
-      TokenNER.merge(@index, new)
-    when String === new
-      TokenNER.merge(@index, TokenNER.process(TSV.new(new, :flatten => true)))
-    end
-  end
-
-  def __define_regexp_hook(name, regexp, *args)
-    @regexps << [regexp, name.to_s]
-  end
-
-  def define_regexp(*args, &block)
-    load_config("__define_regexp_hook", *args, &block)
-  end
-
-  def add_regexp(list = {})
-    @regexps.concat list.collect
-  end
-
-  #{{{ Matching
-  
-  def self.find(index, tokens, longest_match = true)
-    return nil unless index.include? tokens.first
-
-    head = tokens.shift
-    next_index = index[head]
-
-    if tokens.empty?
-      if next_index.include? :END
-        return [next_index[:END], [head]]
-      else
-        tokens.unshift head
-        return nil
-      end
-    else
-
-      return [next_index[:END], [head]] if next_index.include?(:END) and not longest_match
-
-      matches = find(next_index, tokens)
-      if not matches.nil?
-        matches.last.unshift head
-        return matches
-      end
-      
-      return [next_index[:END], [head]] if next_index.include?(:END)
-
-      tokens.unshift head
-      return nil
-    end
-  end
-
-  def extract(text)
-    tokens = TokenNER.tokenize_with_regexps text, @regexps
-
-    matches = {}
-    while tokens.any?
-      while NamedEntity === tokens.first
-        matches[tokens.first.type] ||= []
-        matches[tokens.first.type] << tokens.first
-        tokens.shift
-      end
-
-      new_matches = TokenNER.find(@index, tokens, longest_match) 
-      if new_matches
-        codes, match_tokens = new_matches
-        match = match_tokens.collect{|t| t.original} * " "
-        match.extend NamedEntity
-        match.range = (match_tokens.first.range.begin..match_tokens.last.range.end)
-        codes.each do |code|
-            matches[code] ||= []
-            matches[code] << match
-        end
-      else
-        tokens.shift
-      end
-    end
-
-    matches
-  end
-
-end

data/test/rbbt/ner/test_named_entity.rb

@@ -1,16 +0,0 @@
-require File.dirname(__FILE__) + '/../../test_helper'
-require 'rbbt/ner/named_entity'
-require 'test/unit'
-
-class TestNamedEntity < Test::Unit::TestCase
-
-  def test_annotate
-    str = "CDK5"
-    NamedEntity.annotate str, :gene, 0.9
-
-    assert String === str
-    assert_equal "CDK5", str
-    assert_equal :gene, str.type
-    assert_equal 0.9, str.score
-  end
-end

data/test/rbbt/ner/test_tokenNER.rb

@@ -1,239 +0,0 @@
-require File.dirname(__FILE__) + '/../../test_helper'
-require 'rbbt-util'
-require 'rbbt/ner/tokenNER'
-require 'rbbt/ner/named_entity'
-require 'test/unit'
-
-class TestTokenNER < Test::Unit::TestCase
-
-  def test_tokenize
-    p TokenNER.tokenize('-') 
-    assert_equal ['a' , 'b', ',', 'c'], TokenNER.tokenize('a b, c')
-
-    assert_equal (10..14), TokenNER.tokenize('123456789 12345').last.range
-    assert_equal (0..8), TokenNER.tokenize('123456789 12345').first.range
-
-
-    text = '123456789 12345'
-    assert_equal '12345', text[TokenNER.tokenize('123456789 12345').last.range]
-  end
-
-  def test_tokenize_with_regexp_empty
-    assert_equal ['a' , 'b', ',', 'c'], TokenNER.tokenize_with_regexps('a b, c')
-
-    assert_equal (10..14), TokenNER.tokenize_with_regexps('123456789 12345').last.range
-    assert_equal (0..8), TokenNER.tokenize_with_regexps('123456789 12345').first.range
-
-
-    text = '123456789 12345'
-    assert_equal '12345', text[TokenNER.tokenize_with_regexps('123456789 12345').last.range]
-  end
-
-
-  def test_merge
-    tokens = %w(a b c)
-    index = {'a' => {'b' => {'c' => {:END => ['CODE']}}}}
-
-    assert_equal index, TokenNER.merge({}, TokenNER.index_for_tokens(tokens, 'CODE'))
-  end
-
-  def test_process
-    lexicon =<<-EOF
-C1;a;A;b b
-C2;1;2;3 3;b
-    EOF
-
-    TmpFile.with_file(lexicon) do |file|
-      index = TokenNER.process(TSV.new(file, :sep => ';', :flatten => true))
-
-      assert_equal ['A', 'a', 'b', '1', '2', '3'].sort, index.keys.sort
-      assert_equal [:END], index['a'].keys
-      assert index['b'].keys.include? 'b'
-      assert index['b'].keys.include? :END
-    end
-  end
-
-  def test_find
-    lexicon =<<-EOF
-C1;a;A;b b
-C2;1;2;3 3;b
-    EOF
-
-
-    TmpFile.with_file(lexicon) do |file|
-      index = TokenNER.process(TSV.new(file, :sep => ';', :flatten => true))
-
-      assert TokenNER.find(index, TokenNER.tokenize('a asdf'), false).first.include?   'C1'
-      assert_equal %w(a), TokenNER.find(index, TokenNER.tokenize('a asdf'), false).last
-
-      assert TokenNER.find(index, TokenNER.tokenize('a asdf'), true).first.include?    'C1'
-
-      assert TokenNER.find(index, TokenNER.tokenize('b b asdf'), true).first.include?  'C1'
-      assert_equal %w(b b), TokenNER.find(index, TokenNER.tokenize('b b asdf'), true).last
-
-      assert TokenNER.find(index, TokenNER.tokenize('b b asdf'), false).first.include? 'C2'
-      assert_equal %w(b), TokenNER.find(index, TokenNER.tokenize('b b asdf'), false).last
-
-      assert TokenNER.find(index, TokenNER.tokenize('b asdf'), false).first.include?   'C2'
-    end
-  end
-
-  def test_extract
-    lexicon =<<-EOF
-C1;a;A;b b
-C2;1;2;3 3;b
-    EOF
-
-    TmpFile.with_file(lexicon) do |file|
-      index = TokenNER.new(file, :sep => ';')
-
-      assert index.extract(' asdfa dsf asdf a asdfasdf ').include? 'C1'
-    end
-
-  end
-
-  def test_polysearch_long_match
-    begin
-      require 'rbbt/sources/polysearch'
-    rescue
-      puts "Polysearch is not available. Some test have not ran."
-      assert true
-      return
-    end
-
-    sentence = "mammary and pituitary neoplasms as well as other drug-related mammary/reproductive tissue alterations in females were considered"
-
-    index = TokenNER.new Rbbt.find_datafile('organ')
-    assert index.extract(sentence).include? 'OR00063'
-
-    index = TokenNER.new Rbbt.find_datafile('disease')
-    assert index.extract(sentence).include? 'DID44386'
-
-    index = TokenNER.new Rbbt.find_datafile('disease'), Rbbt.find_datafile('organ')
-    assert index.extract(sentence).include? 'DID44386'
-
-    index = TokenNER.new Rbbt.find_datafile('disease'), Rbbt.find_datafile('organ')
-    assert index.extract(sentence).include? 'DID44386'
-
-    index = TokenNER.new Rbbt.find_datafile('organ')
-    assert index.extract(sentence).include? 'OR00063'
-    index.merge Rbbt.find_datafile('disease')
-    assert ! index.extract(sentence).include?('OR00063')
-    assert index.extract(sentence).include? 'DID44386'
-  end
-
-
-  def __test_polysearch
-    begin
-      require 'rbbt/sources/polysearch'
-    rescue
-      puts "Polysearch is not available. Some test have not ran."
-      assert true
-      return
-    end
-
-    sentence = "The incidence of influenza complications (otitis media, sinusitis, lower respiratory tract infection, bronchitis, or pneumonia) was significantly lower in the oseltamivir group than in the placebo group (0.4% versus 2.6%, p=0.037)."
-
-    index = TokenNER.new Rbbt.find_datafile('organ')
-    assert index.extract(sentence).include? 'OR00068'
-
-    index = TokenNER.new Rbbt.find_datafile('disease')
-    assert index.extract(sentence).include? 'DID44183'
-
-    index = TokenNER.new Rbbt.find_datafile('disease'), Rbbt.find_datafile('organ')
-    assert index.extract(sentence).include? 'DID44183'
-
-    index = TokenNER.new Rbbt.find_datafile('disease'), Rbbt.find_datafile('organ')
-    assert index.extract(sentence).include? 'DID44183'
-
-    index = TokenNER.new Rbbt.find_datafile('organ')
-    assert index.extract(sentence).include? 'OR00068'
-    index.merge Rbbt.find_datafile('disease')
-    assert ! index.extract(sentence).include?('OR00068')
-    assert index.extract(sentence).include? 'DID44183'
-  end
-
-  def test_match_regexp
-    sentence = "The incidence of influenza complications (otitis media, sinusitis, lower respiratory tract infection, bronchitis, or pneumonia) was significantly lower in the oseltamivir group than in the placebo group (0.4% versus 2.6%, p=0.037)."
-
-    matches, chunks = TokenNER.match_regexp(sentence, /[\d\.]+\%/)
-
-    assert matches.include? '0.4%'
-    assert_equal 3, chunks.length
-
-    chunks.each do |chunk, start|
-      assert_equal(sentence[start..(start + chunk.length - 1)], chunk)
-    end
-  end
-
-  def test_match_regexps
-    sentence = "The incidence of influenza complications (otitis media, sinusitis, lower respiratory tract infection, bronchitis, or pneumonia) was significantly lower in the oseltamivir group than in the placebo group (0.4% versus 2.6%, p=0.037)."
-
-    matches, chunks = TokenNER.match_regexps(sentence, [[/[\d\.]+\%/, "percentage"], [/0.[\d]+/, "pvalue"]])
-
-    assert matches.include? '0.4%'
-    assert matches.select{|m| m == '0.4%'}.first.type == "percentage"
-
-    chunks.each do |chunk, start|
-      assert_equal(sentence[start..(start + chunk.length - 1)], chunk)
-    end
-  end
-
-
-  def test_regexp
-    lexicon =<<-EOF
-C1;sinusitis
-C2;FOO
-    EOF
-
-
-    sentence = "The incidence of influenza complications (otitis media, sinusitis, lower respiratory tract infection, bronchitis, or pneumonia) was significantly lower in the oseltamivir group than in the placebo group (0.4% versus 2.6%, p=0.037)."
-
-    TmpFile.with_file(lexicon) do |file|
-      index = TokenNER.new file,  :sep => ';'
-      assert index.extract(sentence).include? 'C1'
-
-      index.add_regexp /[\d\.]+\%/ => "percentage"
-
-      assert index.extract(sentence).include? 'percentage'
-      assert index.extract(sentence)["percentage"].include? '0.4%'
-    end
-
-    TmpFile.with_file(lexicon) do |file|
-      index = TokenNER.new file,  :sep => ';'
-      assert index.extract(sentence).include? 'C1'
-
-      index.define_regexp do
-        percentage /[\d\.]+\%/
-      end
-
-      assert index.extract(sentence).include? 'percentage'
-      assert index.extract(sentence)["percentage"].include? '0.4%'
-    end
-  end
-
-  def test_regexp_captures
-    lexicon =<<-EOF
-C1;sinusitis
-C2;FOO
-    EOF
-
-
-    sentence = "The incidence of influenza complications (otitis media, sinusitis, lower respiratory tract infection, bronchitis, or pneumonia) was significantly lower in the oseltamivir group than in the placebo group (0.4% versus 2.6%, p=0.037)."
-
-    TmpFile.with_file(lexicon) do |file|
-      index = TokenNER.new file,  :sep => ';'
-      assert index.extract(sentence).include? 'C1'
-
-      index.define_regexp do
-        percentage /([\d\.]+)\%/
-      end
-
-      assert index.extract(sentence).include? 'percentage'
-      assert index.extract(sentence)["percentage"].include? '0.4'
-    end
-  end
-
-end
-
-