rbbt 1.0.0

data/install_scripts/wordlists/stopwords

@@ -0,0 +1 @@
+a been get least our them whether about before getting left ourselves then which after being go less out there while again between goes let over these who ago but going like per they whoever all by gone make put this whom almost came got many putting those whose also can gotten may same through why always cannot had maybe saw till will am come has me see to with an could have mine seen too within and did having more shall two without another do he most she unless won't any does her much should until would anybody doing here my so up wouldn't anyhow done him myself some upon yet anyone down his never somebody us you anything each how no someone very your anyway else i none something was are even if not stand we as ever in now such went at every into of sure were away everyone is off take what back everything isn't on than whatever be for it one that what's became from just onto the when because front last or their where 

data/lib/rbbt/bow/bow.rb

@@ -0,0 +1,87 @@
+require 'rbbt'
+require 'stemmer'
+require 'rbbt/util/misc'
+
+# This module provides methods to extract a bag of words (or bag of bigrams)
+# representation for strings of text, and to produce a vector representations
+# of that bag of words for a given list of terms. This BOW representations of
+# the texts is usually first used to build a Dictionary, and then, with the
+# best selection of terms as determined by the Dictionary::TF_IDF.best of
+# Dictionary::KL.best methods, determine the vector representations for that
+# text.
+module BagOfWords
+
+  # Divide the input string into an array of words (sequences of \w characters).
+  # Words are stemmed and filtered to remove stopwords and words with less than
+  # 2 characters. The list of stopwords is a global variable defined in
+  # 'rbbt/util/misc'.
+  def self.words(text)
+    return [] if text.nil?
+    text.scan(/\w+/).
+      collect{|word| word.downcase.stem}.
+      select{|word|  
+      ! $stopwords.include?(word) && 
+        word.length > 2 && 
+        word =~ /[a-z]/
+    }
+  end
+
+  # Take the array of words for the text and form all the bigrams
+  def self.bigrams(text)
+    words = words(text)
+    bigrams = []
+    lastword = nil
+
+    words.each{|word|
+      if lastword
+        bigrams << "#{lastword} #{word}"
+      end
+      lastword = word
+    }
+
+    words + bigrams
+  end
+
+  # Given an array of terms return a hash with the number of appearances of
+  # each term
+  def self.count(terms)
+    count = Hash.new(0)
+    terms.each{|word| count[word] += 1}
+    count
+  end
+
+
+  # Given a string of text find all the words (or bigrams) and return a hash
+  # with their counts
+  def self.terms(text, bigrams = true)
+
+    if bigrams
+      count(bigrams(text))
+    else
+      count(words(text))
+    end
+  end
+
+  # Given a string of text and a list of terms, which may or may not contain
+  # bigrams, return an array with one entry per term which holds the number of
+  # occurrences of each term in the text.
+  def self.features(text, terms, bigrams = nil)
+    bigrams ||= terms.select{|term| term =~ / /}.any?
+    count = bigrams ? count(bigrams(text)) : count(words(text))
+    count.values_at(*terms)
+  end
+end
+
+class String
+  # Shortcut for BagOfWords.words(self)
+  def words
+    BagOfWords.words(self)
+  end
+
+  # Shortcut for BagOfWords.bigrams(self)
+  def bigrams
+    BagOfWords.bigrams(self)
+  end
+end
+
+

data/lib/rbbt/bow/classifier.rb

@@ -0,0 +1,118 @@
+require 'rbbt/bow/bow'
+require 'rsruby'
+
+# This class uses R to build and use classification models. It needs the
+# 'e1071' R package.
+class Classifier
+
+
+  # Given the path to a features file, which specifies a number of instances
+  # along with their classes and features in a tab separated format, it uses R
+  # to build a svm model which is save to file in the path specified as
+  # modelfile.
+  def self.create_model(featuresfile, modelfile, dictfile = nil)
+
+    r = RSRuby.instance
+    r.source(File.join(Rbbt.datadir, 'classifier/R/classify.R'))
+    r.BOW_classification_model(featuresfile, modelfile)
+
+    nil
+  end
+
+  attr_reader :terms
+
+  # Loads an R interpreter which loads the svm model under modelfile.
+  def initialize(modelfile)
+    @r = RSRuby.instance
+    @r.library('e1071')
+    @r.source(File.join(Rbbt.datadir, 'classifier/R/classify.R'))
+
+    @r.load(modelfile)
+
+    @model = @r.svm_model
+    @terms = @r.eval_R("terms = unlist(attr(attr(svm.model$terms,'factors'),'dimnames')[2])")
+  end
+
+  def classify_feature_array(input) #:nodoc:
+    @r.assign('input', input)
+
+    @r.eval_R('input = t(as.data.frame(input))')
+    @r.eval_R('rownames(input) <- NULL')
+    @r.eval_R('colnames(input) <- terms')
+
+    results = @r.eval_R('BOW.classification.classify(svm.model, input, svm.weights)')
+    results.sort.collect{|p| p[1]}
+  end
+
+  def classify_feature_hash(input) #:nodoc:
+    names = []
+    features = []
+    input.each{|name, feats|
+      names << name.to_s
+      features << feats
+    }
+
+    @r.assign('input', features)
+    @r.assign('input.names', names)
+
+    @r.eval_R('input = t(as.data.frame(input))')
+    @r.eval_R('rownames(input) <- input.names')
+    @r.eval_R('colnames(input) <- terms')
+
+    @r.eval_R('BOW.classification.classify(svm.model, input, svm.weights)')
+  end
+
+  def classify_text_array(input) #:nodoc:
+    features = input.collect{|text|
+      BagOfWords.features(text, @terms)
+    }
+
+    classify_feature_array(features)
+  end
+
+  def classify_text_hash(input) #:nodoc:
+    features = {}
+    input.each{|key,text|
+      features[key] = BagOfWords.features(text, @terms)
+    }
+
+    classify_feature_hash(features)
+  end
+
+
+  # This is a polymorphic method. The input variable may be a single input, in
+  # which case the results will be just the class, a hash of inputs, in which
+  # case the result will be a hash with the results for each input, or an
+  # array, in which case the result is an array of the results in the same
+  # order. Each input may also be in the form of a string, in which case it
+  # will be transformed into a feature vector, or an array in which case it
+  # will be considered as an feature vector itself.
+  def classify(input)
+    if input.is_a? String
+      return classify_text_array([input]).first
+    end
+
+
+    if input.is_a? Hash
+      return  {} if input.empty?
+      if input.values.first.is_a? String
+        return classify_text_hash(input)
+      elsif input.values.first.is_a? Array
+        return classify_feature_hash(input)
+      end
+    end
+
+    if input.is_a? Array
+      return  [] if input.empty?
+      if input.first.is_a? String
+        return classify_text_array(input)
+      elsif input.first.is_a? Array
+        return classify_feature_array(input)
+      end
+    end
+
+  end
+
+
+
+end

data/lib/rbbt/bow/dictionary.rb

@@ -0,0 +1,218 @@
+class Dictionary
+  attr_reader :terms
+  def initialize
+    @terms = Hash.new(0)
+  end
+
+  def add(terms, &block)
+    terms.each{|term, count|
+      @terms[term] += count
+    }
+  end
+end
+
+class Dictionary::TF_IDF 
+  attr_reader :terms, :docs, :total_terms, :num_docs
+
+  def initialize(options = {})
+    @term_limit = {
+      :limit => 500_000,
+    }.merge(options)[:limit]
+
+    @terms = Hash.new(0)
+    @docs = Hash.new(0)
+    @num_docs = 0
+    @total_terms = 0
+  end
+
+
+  def add(terms)
+    if @term_limit && @terms.length > @term_limit
+      terms = terms.delete_if{|term, count| !@terms.include? term }
+    end
+
+    terms.each{|term, count|
+      @terms[term] += count
+      @total_terms += count
+      @docs[term]  += 1
+    }
+    @num_docs += 1
+  end
+
+  def df
+    df = Hash.new(0)
+    @docs.each{|term, count|
+     df[term] = count.to_f / @num_docs
+    }
+    df
+  end
+
+  def tf
+    tf = Hash.new(0)
+    @terms.each{|term, count|
+     tf[term] = count.to_f / @total_terms
+    }
+    tf
+  end
+
+  def idf
+    idf = Hash.new(0)
+    num_docs = @num_docs.to_f
+    @docs.each{|term, count|
+     idf[term] = Math::log(num_docs / count)
+    }
+    idf
+  end
+
+  def tf_idf
+    tf_idf = Hash.new(0)
+    num_docs = @num_docs.to_f
+    @docs.each{|term, count|
+     tf_idf[term] = @terms[term].to_f / @total_terms * Math::log(num_docs / count)
+    }
+    tf_idf
+  end
+
+  def best(options = {})
+    hi, low, limit = {
+      :low   => 0,
+      :hi    => 1,
+    }.merge(options).
+    values_at(:hi, :low, :limit)
+
+    num_docs = @num_docs.to_f
+    best = df.select{|term, value|
+      value >= low && value <= hi
+    }.collect{|p| 
+      term     = p.first
+      df_value = p.last
+      [term,
+       @terms[term].to_f / num_docs * Math::log(1.0/df_value)
+      ]
+    }
+    if limit
+      Hash[*best.sort{|a,b| b[1] <=>  a[1]}.slice(0, limit).flatten]
+    else
+      Hash[*best.flatten]
+    end
+  end
+
+  def weights(options = {})
+    best_terms = best(options).keys
+    weights = {}
+
+    num_docs = @num_docs.to_f
+    best_terms.each{|term|
+      weights[term] = Math::log(num_docs / @docs[term])
+    }
+    weights
+  end
+
+end
+
+class Dictionary::KL
+  attr_reader :pos_dict, :neg_dict
+
+  def initialize(options = {})
+    @pos_dict = Dictionary::TF_IDF.new(options)
+    @neg_dict = Dictionary::TF_IDF.new(options)
+  end
+
+  def terms
+    (pos_dict.terms.keys + neg_dict.terms.keys).uniq
+  end
+
+  def add(terms, c)
+    dict = (c == :+ || c == '+' ? @pos_dict : @neg_dict)
+    dict.add(terms)
+  end
+
+  def kl
+    kl = {}
+    pos_df = @pos_dict.df
+    neg_df = @neg_dict.df
+
+    terms.each{|term|
+      pos = pos_df[term]
+      neg = neg_df[term]
+
+      pos = 0.000001 if pos == 0
+      pos = 0.999999 if pos == 1
+      neg = 0.000001 if neg == 0
+      neg = 0.999999 if neg == 1
+
+      kl[term] = pos * Math::log(pos / neg) + neg * Math::log(neg / pos)
+    }
+    kl
+  end
+  
+  def best(options = {})
+    hi, low, limit = {
+      :low   => 0,
+      :hi    => 1,
+    }.merge(options).
+    values_at(:hi, :low, :limit)
+
+    pos_df = @pos_dict.df
+    neg_df = @neg_dict.df
+
+    best = {}
+    terms.select{|term|
+      pos_df[term] >= low && pos_df[term] <= hi ||
+      neg_df[term] >= low && neg_df[term] <= hi 
+    }.each{|term|
+      pos = pos_df[term]
+      neg = neg_df[term]
+
+      pos = 0.000001 if pos == 0
+      pos = 0.999999 if pos == 1
+      neg = 0.000001 if neg == 0
+      neg = 0.999999 if neg == 1
+
+      best[term] = pos * Math::log(pos / neg) + neg * Math::log(neg / pos)
+    }
+    if limit
+      Hash[*best.sort{|a,b| b[1] <=>  a[1]}.slice(0, limit).flatten]
+    else
+      Hash[*best.flatten]
+    end
+  end
+
+  def weights(options = {})
+    best(options)
+  end
+
+   
+    
+end
+
+if __FILE__ == $0
+
+  require 'benchmark'
+  require 'rbbt/sources/pubmed'
+  require 'rbbt/bow/bow'
+  require 'progress-meter'
+
+  max = 10000
+
+  pmids = PubMed.query("Homo Sapiens", max)
+  Progress.monitor "Get pimds"
+  docs = PubMed.get_article(pmids).values.collect{|article| BagOfWords.terms(article.text)}
+
+  dict = Dictionary::TF_IDF.new()
+
+  puts "Starting Benchmark"
+  puts Benchmark.measure{
+    docs.each{|doc|
+      dict.add doc
+    }
+  }
+  puts Benchmark.measure{
+    dict.weights
+  }
+
+  puts dict.terms.length
+
+
+end
+

data/lib/rbbt/ner/abner.rb

@@ -0,0 +1,34 @@
+require 'rbbt'
+require 'rjb'
+
+# Offers a Ruby interface to the Abner Named Entity Recognition Package
+# in Java Abner[http://www.cs.wisc.edu/~bsettles/abner/].
+class Abner
+
+  @@JFile = Rjb::import('java.io.File')
+  @@Tagger = Rjb::import('abner.Tagger')
+  @@Trainer = Rjb::import('abner.Trainer')
+
+  # If modelfile is present a custom trained model can be used,
+  # otherwise, the default BioCreative model is used.
+  def initialize(modelfile=nil)
+    if modelfile == nil         
+      @tagger = @@Tagger.new(@@Tagger.BIOCREATIVE)
+    else                
+      @tagger = @@Tagger.new(@@JFile.new(modelfile))
+    end
+  end
+
+  # Given a chunk of text, it finds all the mentions appearing in it. It
+  # returns all the mentions found, regardless of type, to be coherent
+  # with the rest of NER packages in Rbbt.
+  def extract(text)
+
+    res = @tagger.getEntities(text)
+    types = res[1]
+    strings = res[0]
+
+    return strings.collect{|s| s.to_s}
+  end
+
+end

data/lib/rbbt/ner/banner.rb

@@ -0,0 +1,73 @@
+require 'rbbt'
+require 'rjb'
+
+# Offers a Ruby interface to the Banner Named Entity Recognition Package
+# in Java. Banner[http://banner.sourceforge.net/].
+class Banner
+
+
+  @@JFile = Rjb::import('java.io.File')
+  @@SimpleTokenizer = Rjb::import('banner.tokenization.SimpleTokenizer')
+  @@CRFTagger = Rjb::import('banner.tagging.CRFTagger')
+  @@ParenthesisPostProcessor = Rjb::import('banner.processing.ParenthesisPostProcessor')
+  @@HeppleTagger = Rjb::import('dragon.nlp.tool.HeppleTagger')
+  @@Sentence = Rjb::import('banner.Sentence')
+  @@EngLemmatiser = Rjb::import('dragon.nlp.tool.lemmatiser.EngLemmatiser')
+
+
+
+  # The parameters are set to default values, the only one that one
+  # might want to change is the modelfile to point to a custom trained
+  # one.
+  def initialize(modelfile = File.join(Rbbt.datadir, 'third_party/banner/gene_model.bin'),
+                 lemmadir  = File.join(Rbbt.datadir, 'third_party/banner/nlpdata/lemmatiser'),
+                 taggerdir = File.join(Rbbt.datadir, 'third_party/banner/nlpdata/tagger')
+                )
+
+    @tokenizer = @@SimpleTokenizer.new
+
+    model = @@JFile.new(modelfile)
+    lemma =  @@EngLemmatiser.new(lemmadir,false,true)
+    helper =  @@HeppleTagger.new(taggerdir)
+
+    # The next lines are needed to avoid colisions with
+    # metraprograming that could define load (activesupport in
+    # particular :@ ). RJB seems to call java on method missing
+    class << @@CRFTagger
+      if method_defined? :load 
+        undef_method :load 
+      end
+    end  
+
+    @tagger    = @@CRFTagger.load( model, lemma, helper)
+    @parenPP   = @@ParenthesisPostProcessor.new()
+  end
+
+  
+  # Returns an array with the mention found in the provided piece of
+  # text.
+  def extract(text)
+    text.gsub!(/\n/,' ')
+    text.gsub!(/\|/,'/') # Character | gives an error
+    sentence = @@Sentence.new(text)
+    @tokenizer.tokenize(sentence)
+    @tagger.tag(sentence)
+    @parenPP.postProcess(sentence)
+    tagged = sentence.getSGML
+
+    res = tagged.scan(/<GENE>.*?<\/GENE>/).
+      collect{|r|
+      r.match(/<GENE>(.*?)<\/GENE>/)
+      mention = $1
+      mention.sub!(/^\s*/,'')
+      mention.sub!(/\s*$/,'')
+      mention
+    }
+    res
+  end
+
+
+end
+
+
+

data/lib/rbbt/ner/regexpNER.rb

@@ -0,0 +1,62 @@
+require 'rbbt/util/open'
+require 'rbbt/util/misc'
+
+class RegExpNER
+
+  def self.match_re(text, res)
+    res = [res] unless Array === res
+
+    res.collect{|re|
+      if text.match(re)
+        $1
+      else
+        nil
+      end
+    }.compact
+  end
+
+  def self.build_re(names, ignorecase=true)
+    names.compact.select{|n| n != ""}.
+      sort{|a,b| b.length <=> a.length}.
+      collect{|n| 
+        re = Regexp.quote(n).gsub(/\\?\s/,'\s+')
+        /(?:^|[^\w])(#{ re })(?:$|[^\w])/i
+      }
+  end
+
+  def initialize(lexicon, options = {})
+    options[:flatten] = true
+    options[:ignorecase] = true if options[:ignorecase].nil?
+    options[:stopwords] = true if options[:stopwords].nil?
+
+    data = Open.to_hash(lexicon, options)
+
+    @index = {}
+    data.collect{|code, names|
+      next if code.nil? || code == ""
+      if options[:stopwords]
+        names = names.select{|n| 
+          ! $stopwords.include?(options[:ignorecase] ? n.downcase : n)
+        } 
+      end
+      @index[code] = RegExpNER.build_re(names, options[:ignorecase])
+   }
+  end
+
+  def match_hash(text)
+    matches = {}
+    @index.each{|code, re|
+      RegExpNER.match_re(text, re).each{|match|
+         matches[code] ||= []
+         matches[code] << match
+      }
+    }
+    matches
+  end
+
+  def match(text)
+    match_hash(text).values.flatten
+  end
+
+end
+