rbbt 1.2.5 → 2.0.0

data/install_scripts/wordlists/stopwords

@@ -1 +0,0 @@
-a been get least our them whether about before getting left ourselves then which after being go less out there while again between goes let over these who ago but going like per they whoever all by gone make put this whom almost came got many putting those whose also can gotten may same through why always cannot had maybe saw till will am come has me see to with an could have mine seen too within and did having more shall two without another do he most she unless won't any does her much should until would anybody doing here my so up wouldn't anyhow done him myself some upon yet anyone down his never somebody us you anything each how no someone very your anyway else i none something was are even if not stand we as ever in now such went at every into of sure were away everyone is off take what back everything isn't on than whatever be for it one that what's became from just onto the when because front last or their where 

data/lib/rbbt.rb

@@ -1,83 +0,0 @@
-require 'fileutils'
-require 'yaml'
-
-
-# This module implements a number of utilities aimed at performing Text
-# Mining of BioMedical data. I includes the following:
-#
-# * Multi-purpose Named Entity Recognition and Normalization. And training data for
-#   Gene Mention from the BioCreative competition.
-# * Document Classification
-# * Interfaces to Gene Ontology, Entrez Gene, BioMart and PubMed
-# 
-# There are a number of classes to help gather and integrate the
-# information from all the sources. It is design to be very flexible,
-# but with a sensible set of defaults. 
-#
-module Rbbt
-
-  class NoConfig < Exception; end
-
-  @@rootdir = File.dirname(File.dirname(__FILE__))
-
-  @@datadir = @@cachedir = @@tmpdir = nil
-
-  def self.load_config
-    if File.exist?(File.join(@@rootdir, 'rbbt.config'))
-      config = YAML.load_file(File.join(@@rootdir, 'rbbt.config'))
-      if config.is_a? Hash
-        @@datadir  = config['datadir'] if config['datadir'] 
-        @@cachedir = config['cachedir'] if config['cachedir']
-        @@tmpdir   = config['tmpdir'] if config['tmpdir']
-      end
-    end
-
-
-
-    if File.exist?(File.join(ENV['HOME'], '.rbbt'))
-      config = YAML.load_file(File.join(ENV['HOME'], '.rbbt') )
-      if config.is_a? Hash
-        @@datadir  = config['datadir'] if config['datadir'] 
-        @@cachedir = config['cachedir'] if config['cachedir']
-        @@tmpdir   = config['tmpdir'] if config['tmpdir']
-      end
-    end
-
-    if @@datadir.nil?  || @@cachedir.nil? || @@tmpdir.nil?
-      raise NoConfig, "rbbt not configured. Edit #{File.join(@@rootdir, 'rbbt.config')} or $HOME/.rbbt"
-    end
-
-
-    FileUtils.mkdir_p @@datadir  unless File.exist? @@datadir
-    FileUtils.mkdir_p @@cachedir unless File.exist? @@cachedir
-    FileUtils.mkdir_p @@tmpdir   unless File.exist? @@tmpdir
-
-
-
-    # For some reason banner.jar must be loaded before abner.jar
-    ENV['CLASSPATH'] ||= ""
-    ENV['CLASSPATH'] += ":" + %w(banner abner).collect{|pkg| File.join(datadir, "third_party", pkg, "#{ pkg }.jar")}.join(":")
-  end
-
-  def self.rootdir
-    @@rootdir
-  end 
-
-
-  def self.datadir
-    @@datadir
-  end 
-
-  def self.cachedir
-    @@cachedir
-  end 
-  
-  def self.tmpdir
-    @@tmpdir
-  end 
-
-
-  self.load_config
-end
-
-

data/lib/rbbt/bow/bow.rb

@@ -1,88 +0,0 @@
-require 'rbbt'
-require 'stemmer'
-require 'rbbt/util/misc'
-
-# This module provides methods to extract a bag of words (or bag of bigrams)
-# representation for strings of text, and to produce a vector representations
-# of that bag of words for a given list of terms. This BOW representations of
-# the texts is usually first used to build a Dictionary, and then, with the
-# best selection of terms as determined by the Dictionary::TF_IDF.best of
-# Dictionary::KL.best methods, determine the vector representations for that
-# text.
-module BagOfWords
-
-  # Divide the input string into an array of words (sequences of \w characters).
-  # Words are stemmed and filtered to remove stopwords and words with less than
-  # 2 characters. The list of stopwords is a global variable defined in
-  # 'rbbt/util/misc'.
-  def self.words(text)
-    return [] if text.nil?
-    raise "Stopword list not loaded. Have you installed the wordlists? (rbbt_config prepare wordlists)" if $stopwords.nil?
-    text.scan(/\w+/).
-      collect{|word| word.downcase.stem}.
-      select{|word|  
-      ! $stopwords.include?(word) && 
-        word.length > 2 && 
-        word =~ /[a-z]/
-    }
-  end
-
-  # Take the array of words for the text and form all the bigrams
-  def self.bigrams(text)
-    words = words(text)
-    bigrams = []
-    lastword = nil
-
-    words.each{|word|
-      if lastword
-        bigrams << "#{lastword} #{word}"
-      end
-      lastword = word
-    }
-
-    words + bigrams
-  end
-
-  # Given an array of terms return a hash with the number of appearances of
-  # each term
-  def self.count(terms)
-    count = Hash.new(0)
-    terms.each{|word| count[word] += 1}
-    count
-  end
-
-
-  # Given a string of text find all the words (or bigrams) and return a hash
-  # with their counts
-  def self.terms(text, bigrams = true)
-
-    if bigrams
-      count(bigrams(text))
-    else
-      count(words(text))
-    end
-  end
-
-  # Given a string of text and a list of terms, which may or may not contain
-  # bigrams, return an array with one entry per term which holds the number of
-  # occurrences of each term in the text.
-  def self.features(text, terms, bigrams = nil)
-    bigrams ||= terms.select{|term| term =~ / /}.any?
-    count = bigrams ? count(bigrams(text)) : count(words(text))
-    count.values_at(*terms)
-  end
-end
-
-class String
-  # Shortcut for BagOfWords.words(self)
-  def words
-    BagOfWords.words(self)
-  end
-
-  # Shortcut for BagOfWords.bigrams(self)
-  def bigrams
-    BagOfWords.bigrams(self)
-  end
-end
-
-

data/lib/rbbt/bow/classifier.rb

@@ -1,116 +0,0 @@
-require 'rbbt/bow/bow'
-require 'rsruby'
-
-# This class uses R to build and use classification models. It needs the
-# 'e1071' R package.
-class Classifier
-
-
-  # Given the path to a features file, which specifies a number of instances
-  # along with their classes and features in a tab separated format, it uses R
-  # to build a svm model which is save to file in the path specified as
-  # modelfile.
-  def self.create_model(featuresfile, modelfile, dictfile = nil)
-
-    r = RSRuby.instance
-    r.source(File.join(Rbbt.datadir, 'classifier/R/classify.R'))
-    r.BOW_classification_model(featuresfile, modelfile)
-
-    nil
-  end
-
-  attr_reader :terms
-
-  # Loads an R interpreter which loads the svm model under modelfile.
-  def initialize(modelfile)
-    @r = RSRuby.instance
-    @r.library('e1071')
-    @r.source(File.join(Rbbt.datadir, 'classifier/R/classify.R'))
-
-    @r.load(modelfile)
-
-    @model = @r.svm_model
-    @terms = @r.eval_R("terms = unlist(attr(attr(svm.model$terms,'factors'),'dimnames')[2])")
-  end
-
-  def classify_feature_array(input) #:nodoc:
-    @r.assign('input', input)
-
-    @r.eval_R('input = t(as.data.frame(input))')
-    @r.eval_R('rownames(input) <- NULL')
-    @r.eval_R('colnames(input) <- terms')
-
-    results = @r.eval_R('BOW.classification.classify(svm.model, input, svm.weights)')
-    results.sort.collect{|p| p[1]}
-  end
-
-  def classify_feature_hash(input) #:nodoc:
-    names = []
-    features = []
-    input.each{|name, feats|
-      names << name.to_s
-      features << feats
-    }
-
-    @r.assign('input', features)
-    @r.assign('input.names', names)
-
-    @r.eval_R('input = t(as.data.frame(input))')
-    @r.eval_R('rownames(input) <- input.names')
-    @r.eval_R('colnames(input) <- terms')
-
-    @r.eval_R('BOW.classification.classify(svm.model, input, svm.weights)')
-  end
-
-  def classify_text_array(input) #:nodoc:
-    features = input.collect{|text|
-      BagOfWords.features(text, @terms)
-    }
-
-    classify_feature_array(features)
-  end
-
-  def classify_text_hash(input) #:nodoc:
-    features = {}
-    input.each{|key,text|
-      features[key] = BagOfWords.features(text, @terms)
-    }
-
-    classify_feature_hash(features)
-  end
-
-
-  # This is a polymorphic method. The input variable may be a single input, in
-  # which case the results will be just the class, a hash of inputs, in which
-  # case the result will be a hash with the results for each input, or an
-  # array, in which case the result is an array of the results in the same
-  # order. Each input may also be in the form of a string, in which case it
-  # will be transformed into a feature vector, or an array in which case it
-  # will be considered as an feature vector itself.
-  def classify(input)
-    if input.is_a? String
-      return classify_text_array([input]).first
-    end
-
-
-    if input.is_a? Hash
-      return  {} if input.empty?
-      if input.values.first.is_a? String
-        return classify_text_hash(input)
-      elsif input.values.first.is_a? Array
-        return classify_feature_hash(input)
-      end
-    end
-
-    if input.is_a? Array
-      return  [] if input.empty?
-      if input.first.is_a? String
-        return classify_text_array(input)
-      elsif input.first.is_a? Array
-        return classify_feature_array(input)
-      end
-    end
-
-  end
-
-end

data/lib/rbbt/bow/dictionary.rb

@@ -1,187 +0,0 @@
-class Dictionary
-  attr_reader :terms
-  def initialize
-    @terms = Hash.new(0)
-  end
-
-  def add(terms, &block)
-    terms.each{|term, count|
-      @terms[term] += count
-    }
-  end
-end
-
-class Dictionary::TF_IDF 
-  attr_reader :terms, :docs, :total_terms, :num_docs
-
-  def initialize(options = {})
-    @term_limit = {
-      :limit => 500_000,
-    }.merge(options)[:limit]
-
-    @terms = Hash.new(0)
-    @docs = Hash.new(0)
-    @num_docs = 0
-    @total_terms = 0
-  end
-
-
-  def add(terms)
-    if @term_limit && @terms.length > @term_limit
-      terms = terms.delete_if{|term, count| !@terms.include? term }
-    end
-
-    terms.each{|term, count|
-      @terms[term] += count
-      @total_terms += count
-      @docs[term]  += 1
-    }
-    @num_docs += 1
-  end
-
-  def df
-    df = Hash.new(0)
-    @docs.each{|term, count|
-     df[term] = count.to_f / @num_docs
-    }
-    df
-  end
-
-  def tf
-    tf = Hash.new(0)
-    @terms.each{|term, count|
-     tf[term] = count.to_f / @total_terms
-    }
-    tf
-  end
-
-  def idf
-    idf = Hash.new(0)
-    num_docs = @num_docs.to_f
-    @docs.each{|term, count|
-     idf[term] = Math::log(num_docs / count)
-    }
-    idf
-  end
-
-  def tf_idf
-    tf_idf = Hash.new(0)
-    num_docs = @num_docs.to_f
-    @docs.each{|term, count|
-     tf_idf[term] = @terms[term].to_f / @total_terms * Math::log(num_docs / count)
-    }
-    tf_idf
-  end
-
-  def best(options = {})
-    hi, low, limit = {
-      :low   => 0,
-      :hi    => 1,
-    }.merge(options).
-    values_at(:hi, :low, :limit)
-
-    num_docs = @num_docs.to_f
-    best = df.select{|term, value|
-      value >= low && value <= hi
-    }.collect{|p| 
-      term     = p.first
-      df_value = p.last
-      [term,
-       @terms[term].to_f / num_docs * Math::log(1.0/df_value)
-      ]
-    }
-    if limit
-      Hash[*best.sort{|a,b| b[1] <=>  a[1]}.slice(0, limit).flatten]
-    else
-      Hash[*best.flatten]
-    end
-  end
-
-  def weights(options = {})
-    best_terms = best(options).keys
-    weights = {}
-
-    num_docs = @num_docs.to_f
-    best_terms.each{|term|
-      weights[term] = Math::log(num_docs / @docs[term])
-    }
-    weights
-  end
-
-end
-
-class Dictionary::KL
-  attr_reader :pos_dict, :neg_dict
-
-  def initialize(options = {})
-    @pos_dict = Dictionary::TF_IDF.new(options)
-    @neg_dict = Dictionary::TF_IDF.new(options)
-  end
-
-  def terms
-    (pos_dict.terms.keys + neg_dict.terms.keys).uniq
-  end
-
-  def add(terms, c)
-    dict = (c == :+ || c == '+' ? @pos_dict : @neg_dict)
-    dict.add(terms)
-  end
-
-  def kl
-    kl = {}
-    pos_df = @pos_dict.df
-    neg_df = @neg_dict.df
-
-    terms.each{|term|
-      pos = pos_df[term]
-      neg = neg_df[term]
-
-      pos = 0.000001 if pos == 0
-      pos = 0.999999 if pos == 1
-      neg = 0.000001 if neg == 0
-      neg = 0.999999 if neg == 1
-
-      kl[term] = pos * Math::log(pos / neg) + neg * Math::log(neg / pos)
-    }
-    kl
-  end
-  
-  def best(options = {})
-    hi, low, limit = {
-      :low   => 0,
-      :hi    => 1,
-    }.merge(options).
-    values_at(:hi, :low, :limit)
-
-    pos_df = @pos_dict.df
-    neg_df = @neg_dict.df
-
-    best = {}
-    terms.select{|term|
-      pos_df[term] >= low && pos_df[term] <= hi ||
-      neg_df[term] >= low && neg_df[term] <= hi 
-    }.each{|term|
-      pos = pos_df[term]
-      neg = neg_df[term]
-
-      pos = 0.000001 if pos == 0
-      pos = 0.999999 if pos == 1
-      neg = 0.000001 if neg == 0
-      neg = 0.999999 if neg == 1
-
-      best[term] = pos * Math::log(pos / neg) + neg * Math::log(neg / pos)
-    }
-    if limit
-      Hash[*best.sort{|a,b| b[1] <=>  a[1]}.slice(0, limit).flatten]
-    else
-      Hash[*best.flatten]
-    end
-  end
-
-  def weights(options = {})
-    best(options)
-  end
-
-   
-    
-end