bioroebe 0.10.80

data/lib/bioroebe/biomart/database.rb

@@ -0,0 +1,128 @@
+#!/usr/bin/ruby -w
+# Encoding: UTF-8
+# frozen_string_literal: true
+# =========================================================================== #
+# === Bioroebe::Biomart::Dataset
+#
+# Class representation for a biomart database.
+#
+# Will contain many Biomart::Dataset objects, and belong to
+# a Biomart::Server.
+# =========================================================================== #
+module Bioroebe
+
+module Biomart
+
+class Database # === Bioroebe::Biomart::Dataset
+
+  include Biomart
+
+  # ========================================================================= #
+  # === initialize
+  # ========================================================================= #
+  def initialize( url, args )
+    @url = url or raise ArgumentError, 'must pass :url'
+    unless @url =~ /martservice/
+      @url = @url+'/martservice'
+    end
+    @name         = args['name'] || args[:name]
+    @display_name = args['displayName'] || args[:display_name]
+    @visible      = ( args['visible'] || args[:visible] ) ? true : false
+    reset
+  end
+
+  # ========================================================================= #
+  # === reset
+  # ========================================================================= #
+  def reset
+    @datasets = {}
+  end
+
+  # ========================================================================= #
+  # === visible?
+  # ========================================================================= #
+  def visible?
+    @visible
+  end; alias visible visible? # === visible
+
+  # ========================================================================= #
+  # === list_datasets
+  #
+  # Returns an array of the dataset names (biomart 'name') 
+  # for this dataset.
+  # 
+  # @return [Array] An array of dataset names.
+  # ========================================================================= #
+  def list_datasets
+    if @datasets.empty?
+      fetch_datasets
+    end
+    return @datasets.keys
+  end
+
+  # ========================================================================= #
+  # === name?
+  # ========================================================================= #
+  def name?
+    @name
+  end; alias name name? # === name
+
+  # ========================================================================= #
+  # === datasets?
+  #
+  # Returns a hash (keyed by the biomart 'name' for the dataset) 
+  # of all of the Biomart::Dataset objects belonging to this server.
+  # 
+  # @return [Hash] A hash of Biomart::Dataset objects keyed by 'name'
+  # ========================================================================= #
+  def datasets?
+    if @datasets.empty?
+      fetch_datasets
+    end
+    return @datasets
+  end; alias datasets datasets? # === dataset
+    
+  # ========================================================================= #
+  # === visible?
+  #
+  # Returns true / false if this database is visbile in the 
+  # default MartView interface.
+  # 
+  # @return [Boolean] true/false
+  # ========================================================================= #
+  def visible?
+    @visible
+  end
+
+  # ========================================================================= #
+  # === fetch_datasets
+  #
+  # Utility method to do the webservice call to the biomart server and
+  # collate/build the information about the datasets.
+  # ========================================================================= #
+  def fetch_datasets
+    url = @url + "?type=datasets&mart=#{@name}"
+    document = request( :url => url )
+    tsv_data = []
+    # Ruby >= 1.9 CSV code.
+    tsv_data = CSV.parse( document, { col_sep: "\t" } )
+    tsv_data.each { |t|
+      if t[1] and ( t[3] === '1' )
+        dataset_attr = {
+          'name'         => t[1],
+          'displayName'  => t[2],
+          'visible'      => t[3]
+        }
+        @datasets[ dataset_attr['name'] ] = Dataset.new( @url, dataset_attr )
+      end
+    }
+  end; private :fetch_datasets
+
+  # ========================================================================= #
+  # === display_name?
+  # ========================================================================= #
+  def display_name?
+    @display_name
+  end; alias display_name display_name? # === display_name
+
+end; end; end

data/lib/bioroebe/biomart/dataset.rb

@@ -0,0 +1,572 @@
+#!/usr/bin/ruby -w
+# Encoding: UTF-8
+# frozen_string_literal: true
+# =========================================================================== #
+# === Bioroebe::Biomart::Dataset
+#
+# Class represetation for a biomart dataset.
+#
+# Can belong to a Biomart::Database and a Biomart::Server.
+# =========================================================================== #
+module Bioroebe
+
+module Biomart
+
+class Dataset # === Bioroebe::Biomart::Dataset
+
+  include Biomart
+
+  attr_reader :display_name
+
+  # ========================================================================= #
+  # === initialize
+  #
+  # Creates a new Biomart::Dataset object.
+  #
+  # @param [String] url The URL location of the biomart server.
+  # @param [Hash] args An arguments hash giving details of the dataset.
+  #
+  # arguments hash:
+  #
+  #   {
+  #     :name         => String,     #
+  #     "name"        => String,     #
+  #     :display_name => {}          #
+  #     
+  #   }
+  # ========================================================================= #
+  def initialize(url, args)
+    @url = url or raise ArgumentError, 'must pass :url'
+    unless @url =~ /martservice/
+      @url = @url+'/martservice'
+    end
+
+    @name         = args['name'] || args[:name]
+    @display_name = args['displayName'] || args[:display_name]
+    @visible      = ( args["visible"] || args[:visible] ) ? true : false
+    reset
+  end
+
+  # ========================================================================= #
+  # === reset                                                     (reset tag)
+  # ========================================================================= #
+  def reset
+    # ======================================================================= #
+    # === @filters
+    # ======================================================================= #
+    @filters     = {}
+    # ======================================================================= #
+    # === @attributes
+    # ======================================================================= #
+    @attributes  = {}
+    # ======================================================================= #
+    # === @importables
+    # ======================================================================= #
+    @importables = {}
+    # ======================================================================= #
+    # === @exportables
+    # ======================================================================= #
+    @exportables = {}
+  end
+
+  # ========================================================================= #
+  # === visible?
+  # ========================================================================= #
+  def visible?
+    @visible
+  end; alias visible visible? # === visible
+
+  # ========================================================================= #
+  # === filters?
+  #
+  # Returns a hash (keyed by the biomart 'internal_name' for the filter) 
+  # of all of the Biomart::Filter objects belonging to this dataset.
+  # 
+  # @return [Hash] A hash of Biomart::Filter objects keyed by 'internal_name'
+  # ========================================================================= #
+  def filters?
+    if @filters.empty?
+      fetch_configuration()
+    end
+    return @filters
+  end; alias filters filters? # === filters
+
+  # ========================================================================= #
+  # === list_filters
+  #
+  # Returns an array of the filter names (biomart 'internal_name') 
+  # for this dataset.
+  # 
+  # @return [Array] An array of filters (their 'internal_name's)
+  # ========================================================================= #
+  def list_filters
+    if @filters.empty?
+      fetch_configuration()
+    end
+    return @filters.keys
+  end
+    
+  # ========================================================================= #
+  # === attributes
+  #
+  # Returns a hash (keyed by the biomart 'internal_name' for the attribute) 
+  # of all of the Biomart::Attribute objects belonging to this dataset.
+  # 
+  # @return [Hash] A hash of Biomart::Attribute objects keyed by 'internal_name'
+  # ========================================================================= #
+  def attributes
+    if @attributes.empty?
+      fetch_configuration()
+    end
+    return @attributes
+  end
+
+  # ========================================================================= #
+  # === list_attributes
+  #
+  # Returns an array of the attribute names (biomart 'internal_name') 
+  # for this dataset.
+  # 
+  # @return [Array] An array of attributes (their 'internal_name's)
+  # ========================================================================= #
+  def list_attributes
+    if @attributes.empty?
+      fetch_configuration
+    end
+    return @attributes.keys
+  end
+
+  # ========================================================================= #
+  # === count
+  #
+  # Function to perform a Biomart count. Returns an integer value for the
+  # result of the count query.
+  #
+  # arguments:
+  # 
+  #   {
+  #     :timeout => integer, # set a timeout length for the request (secs) - optional
+  #     :filters => {}       # hash of key-value pairs (filter => search term) - optional
+  #   }
+  # 
+  # @param [Hash] args The arguments hash
+  # @raise Biomart::ArgumentError Raised when un-supported arguments are passed
+  # ========================================================================= #
+  def count(args = {})
+    if args[:federate]
+      raise Biomart::ArgumentError, 'You cannot federate a count query.'
+    end
+    if args[:required_attributes]
+      raise Biomart::ArgumentError,
+        'The :required_attributes option is not allowed on count queries.'
+    end
+    result = request(
+      :method  => 'post',
+      :url     => @url,
+      :timeout => args[:timeout],
+      :query   => generate_xml(
+        :filters    => args[:filters], 
+        :attributes => args[:attributes], 
+        :count      => "1"
+      )
+    )
+    return result.to_i
+  end
+
+  # ========================================================================= #
+  # === search
+  #
+  # Function to perform a Biomart search.
+  #
+  # optional arguments:
+  # 
+  #   {
+  #     :process_results     => true/false,   # convert search results to object
+  #     :timeout             => integer,      # set a timeout length for the request (secs)
+  #     :filters             => {},           # hash of key-value pairs (filter => search term)
+  #     :attributes          => [],           # array of attributes to retrieve
+  #     :required_attributes => [],           # array of attributes that are required
+  #     :federate => [
+  #       {
+  #         :dataset    => Biomart::Dataset, # A dataset object to federate with
+  #         :filters    => {},               # hash of key-value pairs (filter => search term)
+  #         :attributes => []                # array of attributes to retrieve
+  #       }
+  #     ]
+  #   }
+  #
+  # Note, if you do not pass any filters or attributes arguments, the defaults 
+  # for the dataset shall be used.
+  #
+  # Also, using the :required_attributes option - this performs AND logic and will require 
+  # data to be returned in all of the listed attributes in order for it to be returned.
+  #
+  # By default will return a hash with the following:
+  # 
+  #   {
+  #     :headers => [],   # array of headers
+  #     :data    => []    # array of arrays containing search results
+  #   }
+  #
+  # But with the :process_results option will return an array of hashes, 
+  # where each hash represents a row of results (keyed by the attribute name).
+  # 
+  #   @param [Hash] args The arguments hash
+  #   @return [Hash/Array] Will return a hash by default (of unprocessed
+  #                        data), or will return an array of hashes
+  #   @raise Biomart::ArgumentError Raised if incorrect arguments are passed
+  #
+  # ========================================================================= #
+  def search(args = {})
+    if args[:required_attributes] and !args[:required_attributes].is_a?(Array)
+      raise Biomart::ArgumentError, "The :required_attributes option must be passed as an array."
+    end
+    response = request(
+      :method  => 'post',
+      :url     => @url,
+      :timeout => args[:timeout],
+      :query   => generate_xml( process_xml_args(args) )
+    )
+    result = process_tsv(args, response)
+    result = filter_data_rows(args, result) if args[:required_attributes]
+    result = conv_results_to_a_of_h(result) if args[:process_results]
+    return result
+  end
+
+  # ========================================================================= #
+  # === generate_xml
+  #
+  # Utility function to build the Biomart query XML - used by #count and #search.
+  #
+  # @see #count
+  # @see #search
+  # ========================================================================= #
+  def generate_xml(
+      args = {}
+    )
+    biomart_xml = ''
+    xml = Builder::XmlMarkup.new( :target => biomart_xml, :indent => 2 )
+    xml.instruct!
+    xml.declare!( :DOCTYPE, :Query )
+    xml.Query( :virtualSchemaName => "default", :formatter => "TSV", :header => "0", :uniqueRows => "1", :count => args[:count], :datasetConfigVersion => "0.6" ) {
+      dataset_xml( xml, self, { :filters => args[:filters], :attributes => args[:attributes] } )
+      if args[:federate]
+        args[:federate].each { |joined_dataset|
+          unless joined_dataset[:dataset].is_a?(Biomart::Dataset)
+            raise Biomart::ArgumentError, "You must pass a Biomart::Dataset object to the :federate[:dataset] option."
+          end
+          dataset_xml(
+            xml,
+            joined_dataset[:dataset],
+            { :filters => joined_dataset[:filters], :attributes => joined_dataset[:attributes] }
+          )
+        }
+      end
+    }
+    return biomart_xml
+  end
+
+  # ========================================================================= #
+  # === alive?
+  #
+  # Simple heartbeat function to test that a Biomart server is online.
+  #
+  # @return [Boolean] true/false
+  # ========================================================================= #
+  def alive?
+    server = Biomart::Server.new(@url)
+    return server.alive?
+  end
+
+  # ========================================================================= #
+  # === name?
+  # ========================================================================= #
+  def name?
+    @name
+  end; alias name name? # === name
+
+  # ========================================================================= #
+  # === filter_data_rows
+  #
+  # Utility function to remove data rows from a search result that do not
+  # include the :required_attributes.
+  # ========================================================================= #
+  def filter_data_rows( args, result )
+    # ======================================================================= #
+    # Get the list of attributes searched for...
+    # ======================================================================= #
+    attributes = args[:attributes] ? args[:attributes] : []
+    if attributes.empty?
+      self.attributes.each { |name,attribute|
+        if attribute.default?
+          attributes.push(name)
+        end
+      }
+    end
+    # ======================================================================= #
+    # Work out which attribute positions we need to test.
+    # ======================================================================= #
+    positions_to_test = []
+    attributes.each_index { |index|
+      if args[:required_attributes].include?(attributes[index])
+        positions_to_test.push(index)
+      end
+    }
+    # ======================================================================= #
+    # Now go through the results and filter out the unwanted data.
+    # ======================================================================= #
+    filtered_data = []
+    result[:data].each { |data_row|
+      save_row_count = 0
+      positions_to_test.each { |position|
+        save_row_count = save_row_count + 1 unless data_row[position].nil?
+      }
+      if save_row_count == positions_to_test.size
+        filtered_data.push(data_row)
+      end
+    }
+    return {
+      :headers => result[:headers],
+      :data    => filtered_data
+    }
+  end; private :filter_data_rows
+
+  private
+    # ======================================================================= #
+    # === fetch_configuration
+    #
+    # Utility function to retrieve and process the configuration 
+    # xml for a dataset
+    # ======================================================================= #
+    def fetch_configuration
+      url = @url + "?type=configuration&dataset=#{@name}"
+      document = REXML::Document.new( request( :url => url ) )
+      # Filters...
+      ['//FilterDescription','//FilterDescription/Option'].each do |filter_xpath|
+        REXML::XPath.each( document, filter_xpath ) { |f|
+          if f.attributes['displayType'] != nil
+            next if f.attributes['displayType'] == 'container'
+            @filters[ f.attributes['internalName'] ] = Filter.new( f.attributes )
+          elsif f.attributes['pointerFilter'] != nil
+            pointer_filter = Filter.new( f.attributes )
+            @filters[ pointer_filter.name ]           = pointer_filter
+            @filters[ pointer_filter.pointer_filter ] = pointer_filter
+          end
+        }
+      end
+      # Attributes are much simpler...
+      REXML::XPath.each( document, '//AttributeDescription' ) { |a|
+        @attributes[ a.attributes["internalName"] ] = Attribute.new( a.attributes )
+      }
+    end
+
+    # ======================================================================= #
+    # === process_xml_args
+    #
+    # Utility function to process and test the arguments passed for the
+    # xml query.
+    # ======================================================================= #
+    def process_xml_args( args = {} )
+      xml_args = {
+        filters:    args[:filters],
+        attributes: args[:attributes]
+      }
+      if args[:federate]
+        unless args[:federate].is_a?(Array)
+          raise Biomart::ArgumentError,
+            'The :federate option must be passed as an array.'
+        end
+        unless args[:federate].size == 1
+          raise Biomart::ArgumentError,
+            'Sorry, we can only federate two datasets at present. '\
+            'This limitation shall be lifted in version 0.8 of biomart.'
+        end
+        xml_args[:federate] = args[:federate]
+      end
+      return xml_args
+    end
+
+    # ======================================================================= #
+    # === dataset_xml
+    #
+    # Helper function to produce the portion of the biomart xml for 
+    # a dataset query.
+    # ======================================================================= #
+    def dataset_xml( xml, dataset, args )
+      xml.Dataset( :name => dataset.name, :interface => "default" ) {
+        if args[:filters]
+          args[:filters].each { |name,value|
+            # =============================================================== #
+            # We must first check whether the specified filter exists
+            # or whether it does not.
+            # =============================================================== #
+            if dataset.filters[name].nil?
+              raise Biomart::ArgumentError, "The filter '#{name}' does not exist"
+            end
+            if dataset.filters[name].type == 'boolean'
+              value = value.downcase if value.is_a? String
+              if [true,'included','only'].include?(value)
+                xml.Filter( :name => name, :excluded => '0' )
+              elsif [false,'excluded'].include?(value)
+                xml.Filter( :name => name, :excluded => '1' )
+              else
+                raise Biomart::ArgumentError, "The boolean filter '#{name}' can only accept 'true/included/only' or 'false/excluded' arguments."
+              end
+            else
+              value = value.join(',') if value.is_a? Array
+              xml.Filter(
+                :name => name,
+                :value => value
+              )
+            end
+          }
+        else
+          dataset.filters.each { |name,filter|
+            if filter.default?
+              case filter.type
+              when 'boolean'
+                xml.Filter( :name => name, :excluded => filter.default_value )
+              else
+                xml.Filter( :name => name, :value => filter.default_value )
+              end
+            end
+          }
+        end
+        unless args[:count]
+          if args[:attributes]
+            args[:attributes].each { |name|
+              xml.Attribute( :name => name )
+            }
+          else
+            dataset.attributes.each { |name,attribute|
+              if attribute.default?
+                xml.Attribute( :name => name )
+              end
+            }
+          end
+        end
+      }
+    end
+
+    # ======================================================================= #
+    # === process_tsv
+    #
+    # Utility function to transform the tab-separated data retrieved 
+    # from the Biomart search query into a ruby object.
+    # ======================================================================= #
+    def process_tsv( args, tsv )
+      headers     = []
+      parsed_data = []
+      append_header_attributes_for_tsv( headers, self, args[:attributes] )
+      if args[:federate]
+        args[:federate].each { |joined_dataset|
+          append_header_attributes_for_tsv(
+            headers, joined_dataset[:dataset], joined_dataset[:attributes]
+          )
+        }
+      end
+      parsed_data = []
+      begin
+        parsed_data = CSV.parse( tsv, { :col_sep => "\t", :skip_blanks => true } )
+      rescue CSV::MalformedCSVError # Handle malformed .csv files.
+        parsed_data = parse_tsv_line_by_line( headers.size, tsv )
+      end
+      return {
+        :headers => headers,
+        :data    => parsed_data
+      }
+    end
+
+    # ======================================================================= #
+    # === append_header_attributes_for_tsv
+    #
+    # Helper function to append the attribute names to the 'headers' array 
+    # for processing the returned results.
+    # ======================================================================= #
+    def append_header_attributes_for_tsv(headers, dataset, attributes)
+      if attributes
+        attributes.each { |attribute|
+          headers.push(attribute)
+        }
+      else
+        dataset.attributes.each { |name,attribute|
+          if attribute.default?
+            headers.push(name)
+          end
+        }
+      end
+    end
+
+    # ======================================================================= #
+    # === parse_tsv_line_by_line
+    #
+    # Utility function to process TSV formatted data that raises errors.
+    # (Biomart has a habit of serving out this...)
+    #
+    # First attempts to use the CSV modules 'parse_line' function to read
+    # in the data, if that fails, tries to use split to recover the data.
+    # ======================================================================= #
+    def parse_tsv_line_by_line( expected_row_size, tsv )
+      parsed_data = []
+      data_by_line = tsv.split("\n")
+      data_by_line.each { |line|
+        elements = []
+        begin
+          elements = CSV.parse_line( line, { :col_sep => "\t" } ) || []
+        rescue CSV::MalformedCSVError # Handled malformed .csv files.
+          elements = []
+        end
+        if elements.size == 0
+          # ================================================================= #
+          # This is a bad line (causing the above Exception), try and
+          # use split to recover.
+          # ================================================================= #
+          elements = line.split("\t")
+          if line =~ /\t$/
+            # If the last attribute resturn is empty add a nil
+            # value to the array as it would have been missed 
+            # by the split function!
+            elements.push(nil)
+          end
+          # Substitute blank strings for nils
+          elements.map! { |elem|
+            if elem.empty?
+              nil
+            else
+              elem
+            end
+          }
+          # ================================================================= #
+          # Add a safety clause.
+          # ================================================================= #
+          if elements.size === expected_row_size
+            parsed_data.push(elements)
+          end
+        else
+          parsed_data.push(elements)
+        end
+      }
+      return parsed_data
+    end
+
+    # ======================================================================= #
+    # === conv_results_to_a_of_h
+    #
+    # Utility function to quickly convert a search result into an array
+    # of Hashes (keyed by the attribute name) for easier processing -
+    # this is not done by default on all searches as this can cause a
+    # large overhead on big data returns.
+    # ======================================================================= #
+    def conv_results_to_a_of_h( search_results )
+      result_objects = []
+      search_results[:data].each { |row|
+        tmp = {}
+        row.each_index { |index|
+          tmp[ search_results[:headers][index] ] = row[index]
+        }
+        result_objects.push(tmp)
+      }
+      return result_objects
+    end
+
+end; end; end