rbbt-text 0.0.4 → 0.1.0

data/lib/rbbt/bow/bow.rb

@@ -1,4 +1,3 @@
-require 'rbbt'
 require 'rbbt/bow/misc'
 require 'stemmer'
 

data/lib/rbbt/ner/abner.rb

@@ -0,0 +1,37 @@
+require 'rbbt'
+require 'rjb'
+require 'rbbt/ner/named_entity'
+
+# Offers a Ruby interface to the Abner Named Entity Recognition Package
+# in Java Abner[http://www.cs.wisc.edu/~bsettles/abner/].
+class Abner
+
+  Rbbt.add_software "ABNER" => ['','']
+
+  @@JFile   = Rjb::import('java.io.File')
+  @@Tagger  = Rjb::import('abner.Tagger')
+  @@Trainer = Rjb::import('abner.Trainer')
+
+  # If modelfile is present a custom trained model can be used,
+  # otherwise, the default BioCreative model is used.
+  def initialize(modelfile=nil)
+    if modelfile == nil         
+      @tagger = @@Tagger.new(@@Tagger.BIOCREATIVE)
+    else                
+      @tagger = @@Tagger.new(@@JFile.new(modelfile))
+    end
+  end
+
+  # Given a chunk of text, it finds all the mentions appearing in it. It
+  # returns all the mentions found, regardless of type, to be coherent
+  # with the rest of NER packages in Rbbt.
+  def extract(text)
+
+    res = @tagger.getEntities(text)
+    types = res[1]
+    strings = res[0]
+
+    strings.zip(types).collect{|mention, type| mention = mention.to_s; NamedEntity mention, types.to_s; mention}
+  end
+
+end

data/lib/rbbt/ner/banner.rb

@@ -0,0 +1,76 @@
+require 'rbbt'
+require 'rjb'
+require 'rbbt/ner/named_entity'
+
+# Offers a Ruby interface to the Banner Named Entity Recognition Package
+# in Java. Banner[http://banner.sourceforge.net/].
+class Banner
+
+  Rbbt.add_software "BANNER" => ['','']
+
+  @@JFile = Rjb::import('java.io.File')
+  @@SimpleTokenizer = Rjb::import('banner.tokenization.SimpleTokenizer')
+  @@CRFTagger = Rjb::import('banner.tagging.CRFTagger')
+  @@ParenthesisPostProcessor = Rjb::import('banner.processing.ParenthesisPostProcessor')
+  @@HeppleTagger = Rjb::import('dragon.nlp.tool.HeppleTagger')
+  @@Sentence = Rjb::import('banner.Sentence')
+  @@EngLemmatiser = Rjb::import('dragon.nlp.tool.lemmatiser.EngLemmatiser')
+
+
+
+  # The parameters are set to default values, the only one that one
+  # might want to change is the modelfile to point to a custom trained
+  # one.
+  def initialize(modelfile = File.join(Rbbt.find_software('BANNER'), 'gene_model.bin'),
+                 lemmadir  = File.join(Rbbt.find_software('BANNER'), 'nlpdata/lemmatiser'),
+                 taggerdir = File.join(Rbbt.find_software('BANNER'), 'nlpdata/tagger')
+                )
+
+    @tokenizer = @@SimpleTokenizer.new
+
+    model = @@JFile.new(modelfile)
+    lemma =  @@EngLemmatiser.new(lemmadir,false,true)
+    helper =  @@HeppleTagger.new(taggerdir)
+
+    # The next lines are needed to avoid colisions with
+    # metraprograming that could define load (activesupport in
+    # particular :@ ). RJB seems to call java on method missing
+    class << @@CRFTagger
+      if method_defined? :load 
+        undef_method :load 
+      end
+    end  
+
+    @tagger    = @@CRFTagger.load( model, lemma, helper)
+    @parenPP   = @@ParenthesisPostProcessor.new()
+  end
+
+  
+  # Returns an array with the mention found in the provided piece of
+  # text.
+  def extract(text)
+    text.gsub!(/\n/,' ')
+    text.gsub!(/\|/,'/') # Character | gives an error
+    sentence = @@Sentence.new(text)
+    @tokenizer.tokenize(sentence)
+    @tagger.tag(sentence)
+    @parenPP.postProcess(sentence)
+    tagged = sentence.getSGML
+
+    res = tagged.scan(/<GENE>.*?<\/GENE>/).
+      collect{|r|
+      r.match(/<GENE>(.*?)<\/GENE>/)
+      mention = $1
+      mention.sub!(/^\s*/,'')
+      mention.sub!(/\s*$/,'')
+      NamedEntity.annotate mention
+      mention
+    }
+    res
+  end
+
+
+end
+
+
+

data/lib/rbbt/ner/named_entity.rb

@@ -0,0 +1,11 @@
+
+module NamedEntity
+  def self.annotate(string, type = nil, score = nil, range = nil)
+    string.extend NamedEntity
+    string.type = type
+    string.score = score
+    string.range = range
+  end
+
+  attr_accessor :type, :score, :range
+end

data/lib/rbbt/ner/oscar3.rb

@@ -0,0 +1,43 @@
+require 'rbbt'
+require 'rjb'
+require 'libxml'
+require 'rbbt/ner/named_entity'
+require 'rbbt/util/log'
+
+class OSCAR3
+  Rbbt.add_software "OSCAR3" => ['','']
+
+  @@TextToSciXML   = Rjb::import('uk.ac.cam.ch.wwmm.ptclib.scixml.TextToSciXML')
+  @@ProcessingDocumentFactory   = Rjb::import('uk.ac.cam.ch.wwmm.oscar3.recogniser.document.ProcessingDocumentFactory')
+  @@MEMMSingleton = Rjb::import('uk.ac.cam.ch.wwmm.oscar3.recogniser.memm.MEMMSingleton')
+  @@MEMM = @@MEMMSingleton.getInstance();
+
+  def initialize
+  end
+
+  def extract(text, type = "CM")
+    Log.debug "OSCAR3: Finding mentions in #{text}"
+    doc  = @@ProcessingDocumentFactory.getInstance().makeTokenisedDocument(@@TextToSciXML.textToSciXML(text), true, false, false);
+    mentions = []
+    it = doc.getTokenSequences().iterator
+    while it.hasNext do 
+      entities = @@MEMM.findNEs(it.next, text)
+
+      keys = entities.keySet.iterator
+      while keys.hasNext do
+        key = keys.next
+        type, rstart, rend, mention = key.to_string.match(/\[NE:(.*):(.*):(.*):(.*)\]/).values_at(1,2,3,4)
+        score  = entities.get(key)
+
+        NamedEntity.annotate mention, type, score, (rstart..rend)
+        
+        mentions << mention
+      end
+    end
+
+    mentions
+  end
+end
+
+
+

data/share/install/software/ABNER

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+INSTALL_HELPER_FILE="$1"
+RBBT_SOFTWARE_DIR="$2"
+source "$INSTALL_HELPER_FILE"
+
+name='ABNER'
+url="http://pages.cs.wisc.edu/~bsettles/abner/abner.jar"
+
+install_jar "$name" "$url"
+

data/share/install/software/BANNER

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+INSTALL_HELPER_FILE="$1"
+RBBT_SOFTWARE_DIR="$2"
+source "$INSTALL_HELPER_FILE"
+
+name='BANNER'
+url="http://sourceforge.net/projects/banner/files/banner/0.2/BANNER_v02.zip/download"
+
+get_pkg "$name" "$url"
+uncompress_pkg "$name" "$url"
+
+cd `build_dir $name`
+libs=`find libs/ -name "*.jar"`
+mkdir classes
+javac -classpath `echo $libs|sed s/\ /:/g` -d classes `find src/ -name "*.java"`
+cd classes
+for f in ../libs/*.jar; do jar xf "$f";done
+jar cf BANNER.jar *
+PKG_DIR="`opt_dir $name`"
+[ -d "$PKG_DIR" ] || mkdir -p "$PKG_DIR"
+mv BANNER.jar $PKG_DIR/
+cd `build_dir $name`
+cp -R nlpdata/ "$PKG_DIR" 
+
+wget "http://sourceforge.net/projects/banner/files/banner/0.2/gene_model_v02.bin/download" -O "$PKG_DIR/gene_model.bin" 
+
+ln -sf  "$PKG_DIR/BANNER.jar"  "$OPT_JAR_DIR/BANNER.jar"
+clean_build
+
+
+
+
+
+

data/share/install/software/OSCAR3

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+INSTALL_HELPER_FILE="$1"
+RBBT_SOFTWARE_DIR="$2"
+source "$INSTALL_HELPER_FILE"
+
+name='OSCAR3'
+url="http://downloads.sourceforge.net/project/oscar3-chem/oscar3-chem/alpha5/oscar3-a5.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Foscar3-chem%2F&ts=1292333939&use_mirror=switch"
+
+get_pkg "$name" "$url"
+uncompress_pkg "$name" "$url"
+
+PKG_DIR=`opt_dir $name`
+[ -d $PKG_DIR ] || mkdir -p $PKG_DIR
+mv `build_dir $name`/oscar3-a5.jar $PKG_DIR/OSCAR3.jar
+ln -sf  "$PKG_DIR/OSCAR3.jar"  "$OPT_JAR_DIR/OSCAR3.jar"
+clean_build
+
+

data/share/stopwords

@@ -0,0 +1 @@
+a been get least our them whether about before getting left ourselves then which after being go less out there while again between goes let over these who ago but going like per they whoever all by gone make put this whom almost came got many putting those whose also can gotten may same through why always cannot had maybe saw till will am come has me see to with an could have mine seen too within and did having more shall two without another do he most she unless won't any does her much should until would anybody doing here my so up wouldn't anyhow done him myself some upon yet anyone down his never somebody us you anything each how no someone very your anyway else i none something was are even if not stand we as ever in now such went at every into of sure were away everyone is off take what back everything isn't on than whatever be for it one that what's became from just onto the when because front last or their where 

data/test/rbbt/ner/test_abner.rb

@@ -0,0 +1,18 @@
+require File.dirname(__FILE__) + '/../../test_helper'
+require 'rbbt/ner/abner'
+require 'test/unit'
+
+class TestAbner < Test::Unit::TestCase
+
+  def test_extract
+    begin
+      ner = Abner.new
+
+      mentions = ner.extract(" The P-ITIM-compelled multi-phosphoprotein complex binds to and activates SHP-2, which in turn dephosphorylates SHIP and Shc and probably other substrates.")
+      ["SHP-2", "SHIP", "Shc"].each{|mention| 
+        assert(mentions.include? mention)
+      }
+    rescue
+    end
+  end
+end

data/test/rbbt/ner/test_banner.rb

@@ -0,0 +1,20 @@
+require File.dirname(__FILE__) + '/../../test_helper'
+require 'rbbt/ner/banner'
+require 'test/unit'
+
+class TestBanner < Test::Unit::TestCase
+
+  def test_extract
+    begin
+      ner = Banner.new
+
+      mentions = ner.extract(" The P-ITIM-compelled multi-phosphoprotein complex binds to and activates SHP-2, which in turn dephosphorylates SHIP and Shc and probably other substrates.")
+      ["SHP - 2", "SHIP", "Shc"].each{|mention| 
+        assert(mentions.include? mention)
+      }
+    rescue
+      puts $!.message
+      puts $!.backtrace
+    end
+  end
+end

data/test/rbbt/ner/test_named_entity.rb

@@ -0,0 +1,16 @@
+require File.dirname(__FILE__) + '/../../test_helper'
+require 'rbbt/ner/named_entity'
+require 'test/unit'
+
+class TestNamedEntity < Test::Unit::TestCase
+
+  def test_annotate
+    str = "CDK5"
+    NamedEntity.annotate str, :gene, 0.9
+
+    assert String === str
+    assert_equal "CDK5", str
+    assert_equal :gene, str.type
+    assert_equal 0.9, str.score
+  end
+end

data/test/rbbt/ner/test_oscar3.rb

@@ -0,0 +1,28 @@
+require File.dirname(__FILE__) + '/../../test_helper'
+require 'rbbt/ner/oscar3'
+require 'rbbt/util/tmpfile'
+require 'test/unit'
+
+class TestOSCAR3 < Test::Unit::TestCase
+
+
+  def test_extract
+    begin
+      ner = OSCAR3.new
+      str  = "Alternatively, rearrangement of O-(ω-haloalkyl)esters 34 of 2-carboethoxy-N-hydroxypyridine-2-selone affords azonianaphthalenium halides 37 in 79% yield"
+
+      mentions = ner.extract(str)
+      mentions = ner.extract(str)
+      mentions = ner.extract(str)
+      mentions = ner.extract(str)
+      good_mentions = ["2-carboethoxy-N-hydroxypyridine-2-selone", "O-(ω-haloalkyl)esters"]
+
+      good_mentions.each{|mention| 
+        assert(mentions.include? mention)
+      }
+    rescue
+      puts $!.message
+      puts $!.backtrace
+    end
+  end
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: rbbt-text
 version: !ruby/object:Gem::Version 
-  hash: 23
+  hash: 27
   prerelease: false
   segments: 
   - 0
+  - 1
   - 0
-  - 4
-  version: 0.0.4
+  version: 0.1.0
 platform: ruby
 authors: 
 - Miguel Vazquez
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-12-01 00:00:00 +01:00
+date: 2010-12-14 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -32,6 +32,20 @@ dependencies:
         version: "0"
   type: :runtime
   version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: stemmer
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
 description: "Text mining tools: named entity recognition and normalization, document classification, bag-of-words, dictionaries, etc"
 email: miguel.vazquez@fdi.ucm.es
 executables: []
@@ -44,10 +58,22 @@ files:
 - lib/rbbt/bow/bow.rb
 - lib/rbbt/bow/dictionary.rb
 - lib/rbbt/bow/misc.rb
+- lib/rbbt/ner/abner.rb
+- lib/rbbt/ner/banner.rb
+- lib/rbbt/ner/named_entity.rb
+- lib/rbbt/ner/oscar3.rb
 - lib/rbbt/ner/regexpNER.rb
+- share/install/software/ABNER
+- share/install/software/BANNER
+- share/install/software/OSCAR3
+- share/stopwords
 - test/rbbt/bow/test_bow.rb
 - test/rbbt/bow/test_dictionary.rb
 - test/rbbt/bow/test_misc.rb
+- test/rbbt/ner/test_abner.rb
+- test/rbbt/ner/test_banner.rb
+- test/rbbt/ner/test_named_entity.rb
+- test/rbbt/ner/test_oscar3.rb
 - test/rbbt/ner/test_regexpNER.rb
 - test/test_helper.rb
 has_rdoc: true
@@ -88,5 +114,9 @@ test_files:
 - test/rbbt/bow/test_bow.rb
 - test/rbbt/bow/test_dictionary.rb
 - test/rbbt/bow/test_misc.rb
+- test/rbbt/ner/test_abner.rb
+- test/rbbt/ner/test_banner.rb
+- test/rbbt/ner/test_named_entity.rb
+- test/rbbt/ner/test_oscar3.rb
 - test/rbbt/ner/test_regexpNER.rb
 - test/test_helper.rb