rbbt 1.2.5 → 2.0.0

data/lib/rbbt/sources/organism.rb

@@ -1,241 +0,0 @@
-require 'rbbt'
-require 'rbbt/util/open'
-require 'rbbt/util/index'
-
-# This module contains some Organism centric functionalities. Each organism is
-# identified by a keyword.
-module Organism
-
-  # Raised when trying to access information for an organism that has not been
-  # prepared already.
-  class OrganismNotProcessedError < StandardError; end
-
-  # Return the list of all supported organisms. The prepared flag is used to
-  # show only those that have been prepared.
-  def self.all(prepared = true)
-    if prepared
-      Dir.glob(File.join(Rbbt.datadir,'/organisms/') + '/*/identifiers').collect{|f| File.basename(File.dirname(f))}
-    else
-      Dir.glob(File.join(Rbbt.datadir,'/organisms/') + '/*').select{|f| File.directory? f}.collect{|f| File.basename(f)}
-    end
-  end
-
-
-  # Return the complete name of an organism. The org parameter is the organism
-  # keyword
-  def self.name(org)
-    raise OrganismNotProcessedError, "Missing 'name' file" if ! File.exists? File.join(Rbbt.datadir,"organisms/#{ org }/name")
-    Open.read(File.join(Rbbt.datadir,"organisms/#{ org }/name"))
-  end
-
-  # Hash linking all the organism log names with their keywords in Rbbt. Its
-  # the inverse of the name method.
-  NAME2ORG = {}
-  Organism::all.each{|org|  
-    name = Organism.name(org).strip.downcase
-    NAME2ORG[name] = org
-  }
-
-
-  # Return the key word associated with an organism.
-  def self.name2org(name)  
-    NAME2ORG[name.strip.downcase]
-  end
-
-  # FIXME: The NER related stuff is harder to install, thats why we hide the
-  # requires next to where they are needed, next to options
-  
-  # Return a NER object which could be of RNER, Abner or Banner class, this is
-  # selected using the type parameter. 
-  def self.ner(org, type=:rner, options = {})
-
-    case type.to_sym
-    when :abner
-      require 'rbbt/ner/abner'
-      return Abner.new
-    when :banner
-      require 'rbbt/ner/banner'
-      return Banner.new
-    when :rner
-      require 'rbbt/ner/rner'
-      model = options[:model] 
-      model ||= File.join(Rbbt.datadir,"ner/model/#{ org }") if File.exist? File.join(Rbbt.datadir,"ner/model/#{ org }")
-      model ||= File.join(Rbbt.datadir,'ner/model/BC2')
-      return NER.new(model)
-    else
-      raise "Ner type (#{ type }) unknown"
-    end
-
-  end
-
-  # Return a normalization object.
-  def self.norm(org, to_entrez = nil)
-    require 'rbbt/ner/rnorm'
-    if to_entrez.nil?
-      to_entrez = id_index(org, :native => 'Entrez Gene Id', :other => [supported_ids(org).first])
-    end
-    
-    token_file = File.join(Rbbt.datadir, 'norm','config',org.to_s + '.config')
-    if !File.exists? token_file
-      token_file = nil
-    end
-
-    Normalizer.new(File.join(Rbbt.datadir,"organisms/#{ org }/lexicon"), :to_entrez => to_entrez, :file => token_file, :max_candidates => 20)
-  end
-
-  # Returns a hash with the names associated with each gene id. The ids are
-  # in Rbbt native format for that organism.
-  def self.lexicon(org, options = {})
-    options = {:sep => "\t|\\|", :flatten => true}.merge(options)
-    Open.to_hash(File.join(Rbbt.datadir,"organisms/#{ org }/lexicon"),options)
-  end
-
-  # Returns a hash with the list of go terms for each gene id. Gene ids are in
-  # Rbbt native format for that organism.
-  def self.goterms(org)
-    Open.to_hash(File.join(Rbbt.datadir,"organisms/#{ org }/gene.go"), :flatten => true)
-  end
-
-  # Return list of PubMed ids associated to the organism. Determined using a
-  # PubMed query with the name of the organism
-  def self.literature(org)
-    Open.read(File.join(Rbbt.datadir,"organisms/#{ org }/all.pmid")).scan(/\d+/)
-  end
-
-  # Return hash that associates genes to a list of PubMed ids.
-  def self.gene_literature(org)
-    Open.to_hash(File.join(Rbbt.datadir,"organisms/#{ org }/gene.pmid"), :flatten => true)
-  end
-
-  # Return hash that associates genes to a list of PubMed ids. Includes only
-  # those found to support GO term associations.
-  def self.gene_literature_go(org)
-    Open.to_hash(File.join(Rbbt.datadir,"organisms/#{ org }/gene_go.pmid"), :flatten => true)
-  end
-
-  # Returns a list with the names of the id formats supported for an organism.
-  # If examples are produced, the list is of [format, example] pairs.
-  # 
-  # *Options:*
-  #
-  # *examples:* Include example ids for each format
-  def self.supported_ids(org, options = {})
-    formats  = []
-    examples = [] if options[:examples]
-    i= 0
-    Open.read(File.join(Rbbt.datadir,"organisms/#{ org }/identifiers")).each_line{|l|
-      if i == 0
-        i += 1
-        next unless l=~/^\s*#/
-        formats  = Open.fields(l.sub(/^[\s#]+/,'')).collect{|n| n.strip}
-        return formats unless examples
-        next
-      end
-
-      if Open.fields(l).select{|name| name && name =~ /\w/}.length > examples.compact.length
-        examples = Open.fields(l).collect{|name| name.split(/\|/).first}
-      end
-      i += 1
-    }
-
-    formats.zip(examples)
-  end
-
-  # Creates a hash where each possible id is associated with the names of the
-  # formats (its potentially possible for different formats to have the same
-  # id). This is used in the guessIdFormat method. 
-  def self.id_formats(org) 
-    id_types = {} 
-    formats = supported_ids(org)
-
-    text = Open.read(File.join(Rbbt.datadir,"organisms/#{ org }/identifiers"))
-    
-    if text.respond_to? :collect
-      lines = text.collect
-    else
-      lines = text.lines
-    end
-
-    lines.each{|l|
-      ids_per_type = Open.fields(l)
-      formats.zip(ids_per_type).each{|p|
-        format = p[0]
-        p[1] ||= ""
-        ids = p[1].split(/\|/)
-        ids.each{|id|
-          next if id.nil? || id == ""
-          id_types[id.downcase] ||= []
-          id_types[id.downcase] << format unless id_types[id.downcase].include? format
-        }
-      }
-    }
-
-    return id_types
-  end
-
-  def self.guessIdFormat(formats, query)
-    query = query.compact.collect{|gene| gene.downcase}.uniq
-    if String === formats
-      formats = id_formats(formats)
-    end
-
-    return nil if formats.values.empty?
-    values = formats.values_at(*query)
-    return nil if values.empty?
-    
-    format_count = {}
-    values.compact.collect{|types| types.uniq}.flatten.each{|f| 
-      format_count[f] ||= 0
-      format_count[f] += 1
-    }
-    
-    return nil if format_count.values.empty?
-    format_count.select{|k,v| v > (query.length / 10)}.sort{|a,b| b[1] <=> a[1]}.first
-  end
-
-  def self.id_position(supported_ids, id_name, options = {})
-    pos = 0
-    supported_ids.each_with_index{|id, i| 
-      if id.strip == id_name.strip || !options[:case_sensitive] && id.strip.downcase == id_name.strip.downcase
-        pos = i; 
-      end
-    }
-    pos
-  end
-
-  def self.id_index(org, options = {})
-    native = options[:native]
-    other  = options[:other]
-    options[:case_sensitive] = false if options[:case_sensitive].nil?
-
-    if native.nil? and other.nil?
-      Index.index(File.join(Rbbt.datadir,"organisms/#{ org }/identifiers"), options)
-    else
-      supported = Organism.supported_ids(org)
-
-      first = nil
-      if native
-        first = id_position(supported,native,options)
-        raise "No match for native format '#{ native }'" if first.nil?
-      else
-        first = 0
-      end
-
-      rest = nil
-      if other
-        rest = other.collect{|name| id_position(supported,name, options)}
-      else
-        rest = (0..supported.length - 1).to_a - [first]
-      end
-
-      options[:native] = first
-      options[:extra] = rest
-      options[:sep] = "\t"
-      index = Index.index(File.join(Rbbt.datadir,"organisms/#{ org }/identifiers"), options)
-
-      index
-    end
-  end
-
-end
-

data/lib/rbbt/sources/polysearch.rb

@@ -1,117 +0,0 @@
-require 'rbbt'
-require 'rbbt/util/open'
-require 'rbbt/ner/regexpNER'
-require 'rbbt/ner/dictionaryNER'
-
-# Find terms in the Polysearch thesauri using simple regular expression
-# matching. Note that the first time the methods are used the correspondent
-# thesaurus are loaded into memory. The available thesauri are: disease, drug,
-# metabolite, organ, subcellular (subcellular localization) and tissue.
-module Polysearch
-
-  
-  @@names = {}
-  def self.type_names(type) #:nodoc:
-    @@names[type] ||= Open.to_hash(File.join(Rbbt.datadir,'dbs','polysearch',type.to_s + '.txt'), :single => true)
-  end
-
-
-  @@indexes = {}
-  def self.type_index(type) #:nodoc:
-    if $stopwords
-      stopwords = $stopwords
-    else
-      stopwords = []
-    end
-
-    case type.to_sym
-    when :disease
-      stopwords << 'use'
-    end
-
-    @@indexes[type] ||= RegExpNER.new(File.join(Rbbt.datadir,'dbs','polysearch',type.to_s + '.txt'), :stopwords => stopwords)
-  end
-
-  # Find matches in a string of text, the types array specifies which thesauri
-  # to use, if if nil it will use all.
-  def self.match(text, types = nil)
-    if types.nil? 
-      types = %w(disease  drug  metabolite  organ  subcellular  tissue)
-    end
-
-    types = [types] unless Array === types
-    types = types.sort
-
-    matches = {}
-    types.collect{|type|
-      matches.merge!(type_index(type).match(text))
-    }
-
-    matches
-  end
-
-  # Transform the code into a name, type is the thesaurus to use
-  def self.name(type, code)
-    type_names(type)[code]
-  end
-
-end
-
-if __FILE__ == $0
-    text =<<-EOT
-
-     Background  Microorganisms adapt their transcriptome by integrating
-     multiple chemical and physical signals from their environment. Shake-flask
-     cultivation does not allow precise manipulation of individual culture
-     parameters and therefore precludes a quantitative analysis of the
-     (combinatorial) influence of these parameters on transcriptional
-     regulation. Steady-state chemostat cultures, which do enable accurate
-     control, measurement and manipulation of individual cultivation parameters
-     (e.g. specific growth rate, temperature, identity of the growth-limiting
-     nutrient) appear to provide a promising experimental platform for such a
-     combinatorial analysis. Results  A microarray compendium of 170
-     steady-state chemostat cultures of the yeast Saccharomyces cerevisiae is
-     presented and analyzed. The 170 microarrays encompass 55 unique
-     conditions, which can be characterized by the combined settings of 10
-     different cultivation parameters. By applying a regression model to assess
-     the impact of (combinations of) cultivation parameters on the
-     transcriptome, most S. cerevisiae genes were shown to be influenced by
-     multiple cultivation parameters, and in many cases by combinatorial
-     effects of cultivation parameters. The inclusion of these combinatorial
-     effects in the regression model led to higher explained variance of the
-     gene expression patterns and resulted in higher function enrichment in
-     subsequent analysis. We further demonstrate the usefulness of the
-     compendium and regression analysis for interpretation of shake-flask-based
-     transcriptome studies and for guiding functional analysis of
-     (uncharacterized) genes and pathways. Conclusions  Modeling the
-     combinatorial effects of environmental parameters on the transcriptome is
-     crucial for understanding transcriptional regulation. Chemostat
-     cultivation offers a powerful tool for such an approach. Keywords:
-     chemostat steady state samples Cerebellar stroke syndrome
-       
-    
-    EOT
-
-    require 'benchmark'
-    require 'ruby-prof'
-
-    puts Benchmark.measure{
-      p Polysearch.match(text,'disease')
-    }
-
-
-    RubyProf.start
-
-    Polysearch.match(text,'disease')
-
-    result = RubyProf.stop
-
-    # Print a flat profile to text
-    printer = RubyProf::FlatPrinter.new(result)
-    printer.print(STDOUT, 0)
-
-    puts Benchmark.measure{
-      10.times{ p Polysearch.match(text,'disease') }
-    }
- 
-end

data/lib/rbbt/sources/pubmed.rb

@@ -1,248 +0,0 @@
-require 'rbbt/util/filecache'
-require 'rbbt/util/open'
-require 'rbbt/sources/gscholar'
-require 'rbbt'
-require 'libxml'
-
-# This module offers an interface with PubMed, to perform queries, and
-# retrieve simple information from articles. It uses the caching
-# services of Rbbt.
-module PubMed
-
-  private
-  @@last = Time.now
-  @@pubmed_lag = 1
-  def self.get_online(pmids)
-
-    pmid_list = ( pmids.is_a?(Array) ? pmids.join(',') : pmids.to_s )
-    url = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=pubmed&retmode=xml&id=#{pmid_list}" 
-
-    diff = Time.now - @@last
-    sleep @@pubmed_lag - diff unless diff > @@pubmed_lag
-
-    xml = Open.read(url, :quiet => true, :nocache => true)
-    
-    @@last = Time.now
-
-    articles = xml.scan(/(<PubmedArticle>.*?<\/PubmedArticle>)/smu).flatten
-
-    if pmids.is_a? Array
-      list = {}
-      articles.each{|article|
-        pmid = article.scan(/<PMID>(.*?)<\/PMID>/).flatten.first
-        list[pmid] = article
-      }
-      return list
-    else
-      return articles.first
-    end
-
-  end
-
-  public
-
-  # Processes the xml with an articles as served by MedLine and extracts
-  # the abstract, title and journal information
-  class Article
-
-
-    XML_KEYS = [
-      [:title    , "ArticleTitle"],
-      [:journal  , "Journal/Title"],
-      [:issue    , "Journal/JournalIssue/Issue"],
-      [:volume   , "Journal/JournalIssue/Volume"],
-      [:issn     , "Journal/ISSN"],
-      [:year     , "Journal/JournalIssue/PubDate/Year"],
-      [:month    , "Journal/JournalIssue/PubDate/Month"],
-      [:pages    , "Pagination/MedlinePgn"],
-      [:abstract , "Abstract/AbstractText"],
-    ]
-
-    PMC_PDF_URL = "http://www.ncbi.nlm.nih.gov/pmc/articles/PMCID/pdf/"
-
-    def self.escape_title(title)
-      title.gsub(/(\w*[A-Z][A-Z]+\w*)/, '{\1}')
-    end
-
-    def self.make_bibentry(lastname, year, title)
-      words = title.downcase.scan(/\w+/)
-      if words.first.length > 3
-        abrev = words.first
-      else
-        abrev = words[0..2].collect{|w| w.chars.first} * ""
-      end
-      [lastname.gsub(/\s/,'_'), year || "NOYEAR", abrev] * ""
-    end
-    def self.parse_xml(xml)
-      parser  = LibXML::XML::Parser.string(xml)
-      pubmed  = parser.parse.find("/PubmedArticle").first
-      medline = pubmed.find("MedlineCitation").first
-      article = medline.find("Article").first
-
-      info = {}
-
-      info[:pmid] = medline.find("PMID").first.content
-
-      XML_KEYS.each do |p|
-        name, key = p
-        node = article.find(key).first
-
-        next if node.nil?
-
-        info[name] = node.content
-      end
-
-      bibentry = nil
-      info[:author] = article.find("AuthorList/Author").collect do |author|
-        begin
-          lastname = author.find("LastName").first.content
-          if author.find("ForeName").first.nil?
-            forename = nil
-          else
-            forename = author.find("ForeName").first.content.split(/\s/).collect{|word| if word.length == 1; then word + '.'; else word; end} * " "
-          end
-          bibentry ||= make_bibentry lastname, info[:year], info[:title]
-        rescue
-        end
-        [lastname, forename] * ", "
-      end * " and "
-
-      info[:bibentry] = bibentry.downcase if bibentry
-
-      info[:pmc_pdf] = pubmed.find("PubmedData/ArticleIdList/ArticleId").select{|id| id[:IdType] == "pmc"}.first
-
-      if info[:pmc_pdf]
-        info[:pmc_pdf] = PMC_PDF_URL.sub(/PMCID/, info[:pmc_pdf].content)
-      end
-
-      info
-    end
-
-    attr_accessor :title, :abstract, :journal, :author, :pmid, :bibentry, :pmc_pdf, :gscholar_pdf, :pdf_url
-    attr_accessor *XML_KEYS.collect{|p| p.first }
-
-    def initialize(xml)
-      if xml && ! xml.empty?
-        info = PubMed::Article.parse_xml xml
-        info.each do |key, value|
-          self.send("#{ key }=", value)
-        end
-      end
-    end
-
-    def pdf_url
-      return pmc_pdf if pmc_pdf
-      @gscholar_pdf ||= GoogleScholar::full_text_url title
-    end
-
-    def full_text
-      return nil if pdf_url.nil?
-
-      text = nil
-      TmpFile.with_file do |pdf|
-
-        # Change user-agent, oh well...
-        `wget --user-agent=firefox #{ pdf_url } -O #{ pdf }`
-        TmpFile.with_file do |txt|
-          `pdftotext #{ pdf } #{ txt }`
-          text = Open.read(txt) if File.exists? txt
-        end
-      end
-
-      text
-    end
-
-    def bibtex
-      keys = [:author] + XML_KEYS.collect{|p| p.first } - [:bibentry]
-      bibtex = "@article{#{bibentry},\n"
-
-      keys.each do |key|
-        next if self.send(key).nil?
-
-        case key
-
-        when :title
-          bibtex += "  title = { #{ PubMed::Article.escape_title title } },\n"
-
-        when :issue
-          bibtex += "  number = { #{ issue } },\n"
-
-        else
-          bibtex += "  #{ key } = { #{ self.send(key) } },\n"
-        end
-
-      end
-
-      bibtex += "  fulltext = { #{ pdf_url } },\n" if pdf_url
-      bibtex += "  pmid = { #{ pmid } }\n}"
-
-
-      bibtex
-    end
-
-    # Join the text from title and abstract
-    def text
-      [title, abstract].join("\n")
-    end
-  end
-
-  # Returns the Article object containing the information for the PubMed
-  # ID specified as an argument. If +pmid+ is an array instead of a single
-  # identifier it returns an hash with the Article object for each id.
-  # It uses the Rbbt cache to save the articles xml.
-  def self.get_article(pmid)
-
-    if pmid.is_a? Array
-      missing = []
-      list = {}
-
-      pmid.each{|p|
-        filename = p.to_s + '.xml'
-        if File.exists? FileCache.path(filename)
-          list[p] = Article.new(Open.read(FileCache.path(filename)))
-        else
-          missing << p
-        end
-      }
-
-      return list unless missing.any?
-      chunk_size = [100, missing.length].min
-      chunks = (missing.length.to_f / chunk_size).ceil
-
-      articles = {}
-      chunks.times do |chunk|
-        pmids = missing[(chunk * chunk_size)..((chunk + 1) *chunk_size)]
-        articles.merge!(get_online(pmids))
-      end
-
-      articles.each{|p, xml|
-        filename = p + '.xml'
-        FileCache.add_file(filename,xml, :force => true)
-        list[p] =  Article.new(xml)
-      }
-
-      return list
-
-    else
-      filename = pmid.to_s + '.xml'
-
-      if File.exists? FileCache.path(filename)
-        return Article.new(Open.read(FileCache.path(filename)))
-      else
-        xml = get_online(pmid)
-        FileCache.add_file(filename,xml)
-
-        return Article.new(xml)
-      end
-    end
-  end
-
-  # Performs the specified query and returns an array with the PubMed
-  # Ids returned. +retmax+ can be used to limit the number of ids
-  # returned, if is not specified 30000 is used.
-  def self.query(query, retmax=nil)
-    retmax ||= 30000
-
-    Open.read("http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?retmax=#{retmax}&db=pubmed&term=#{query}",:quiet => true, :nocache => true).scan(/<Id>(\d+)<\/Id>/).flatten
-  end
-end