lazar 0.0.1

data/lib/algorithm.rb

@@ -0,0 +1,21 @@
+module OpenTox
+
+  module Algorithm 
+
+    # Generic method to execute algorithms
+    # Algorithms should:
+    #   - accept a Compound, an Array of Compounds or a Dataset as first argument
+    #   - optional parameters as second argument
+    #   - return an object corresponding to the input type as result (eg. Compound -> value, Array of Compounds -> Array of values, Dataset -> Dataset with values
+    # @param [OpenTox::Compound,Array,OpenTox::Dataset] Input object
+    # @param [Hash] Algorithm parameters
+    # @return Algorithm result
+    def self.run algorithm, object, parameters=nil
+      bad_request_error "Cannot run '#{algorithm}' algorithm. Please provide an OpenTox::Algorithm." unless algorithm =~ /^OpenTox::Algorithm/
+      klass,method = algorithm.split('.')
+      parameters.nil? ?  Object.const_get(klass).send(method,object) : Object.const_get(klass).send(method,object, parameters)
+    end
+
+  end
+end
+

data/lib/bbrc.rb

@@ -0,0 +1,165 @@
+module OpenTox
+  module Algorithm
+    class Fminer
+      TABLE_OF_ELEMENTS = [
+"H", "He", "Li", "Be", "B", "C", "N", "O", "F", "Ne", "Na", "Mg", "Al", "Si", "P", "S", "Cl", "Ar", "K", "Ca", "Sc", "Ti", "V", "Cr", "Mn", "Fe", "Co", "Ni", "Cu", "Zn", "Ga", "Ge", "As", "Se", "Br", "Kr", "Rb", "Sr", "Y", "Zr", "Nb", "Mo", "Tc", "Ru", "Rh", "Pd", "Ag", "Cd", "In", "Sn", "Sb", "Te", "I", "Xe", "Cs", "Ba", "La", "Ce", "Pr", "Nd", "Pm", "Sm", "Eu", "Gd", "Tb", "Dy", "Ho", "Er", "Tm", "Yb", "Lu", "Hf", "Ta", "W", "Re", "Os", "Ir", "Pt", "Au", "Hg", "Tl", "Pb", "Bi", "Po", "At", "Rn", "Fr", "Ra", "Ac", "Th", "Pa", "U", "Np", "Pu", "Am", "Cm", "Bk", "Cf", "Es", "Fm", "Md", "No", "Lr", "Rf", "Db", "Sg", "Bh", "Hs", "Mt", "Ds", "Rg", "Cn", "Uut", "Fl", "Uup", "Lv", "Uus", "Uuo"]
+        
+      #
+      # Run bbrc algorithm on dataset
+      #
+      # @param [OpenTox::Dataset] training dataset
+      # @param [optional] parameters BBRC parameters, accepted parameters are
+      #   - min_frequency  Minimum frequency (default 5)
+      #   - feature_type Feature type, can be 'paths' or 'trees' (default "trees")
+      #   - backbone BBRC classes, pass 'false' to switch off mining for BBRC representatives. (default "true")
+      #   - min_chisq_significance Significance threshold (between 0 and 1)
+      #   - nr_hits Set to "true" to get hit count instead of presence
+      #   - get_target Set to "true" to obtain target variable as feature
+      # @return [OpenTox::Dataset] Fminer Dataset
+      def self.bbrc training_dataset, params={}
+
+        time = Time.now
+        bad_request_error "More than one prediction feature found in training_dataset #{training_dataset.id}" unless training_dataset.features.size == 1
+
+        prediction_feature = training_dataset.features.first
+        if params[:min_frequency]
+          minfreq = params[:min_frequency]
+        else
+          per_mil = 5 # value from latest version
+          per_mil = 8 # as suggested below
+          i = training_dataset.feature_ids.index prediction_feature.id
+          nr_labeled_cmpds = training_dataset.data_entries.select{|de| !de[i].nil?}.size
+          minfreq = per_mil * nr_labeled_cmpds.to_f / 1000.0 # AM sugg. 8-10 per mil for BBRC, 50 per mil for LAST
+          minfreq = 2 unless minfreq > 2
+          minfreq = minfreq.round
+        end
+
+        @bbrc ||= Bbrc::Bbrc.new
+        @bbrc.Reset
+        if prediction_feature.numeric 
+          @bbrc.SetRegression(true) # AM: DO NOT MOVE DOWN! Must happen before the other Set... operations!
+        else
+          bad_request_error "No accept values for "\
+                            "dataset '#{training_dataset.id}' and "\
+                            "feature '#{prediction_feature.id}'" unless prediction_feature.accept_values
+          value2act = Hash[[*prediction_feature.accept_values.map.with_index]]
+        end
+        @bbrc.SetMinfreq(minfreq)
+        @bbrc.SetType(1) if params[:feature_type] == "paths"
+        @bbrc.SetBackbone(false) if params[:backbone] == "false"
+        @bbrc.SetChisqSig(params[:min_chisq_significance].to_f) if params[:min_chisq_significance]
+        @bbrc.SetConsoleOut(false)
+
+        params[:nr_hits] ? nr_hits = params[:nr_hits] : nr_hits = false
+        feature_dataset = FminerDataset.new(
+            :training_dataset_id => training_dataset.id,
+            :training_algorithm => "#{self.to_s}.bbrc",
+            :training_feature_id => prediction_feature.id ,
+            :training_parameters => {
+              :min_frequency => minfreq,
+              :nr_hits => nr_hits,
+              :backbone => (params[:backbone] == false ? false : true) 
+            }
+
+        )
+        feature_dataset.compounds = training_dataset.compounds
+
+        # add data 
+        training_dataset.compounds.each_with_index do |compound,i|
+          act = value2act[training_dataset.data_entries[i].first]
+          if act # TODO check if this works
+            @bbrc.AddCompound(compound.smiles,i+1)
+            @bbrc.AddActivity(act,i+1)
+          end
+        end
+        #g_median=@fminer.all_activities.values.to_scale.median
+
+        #task.progress 10
+        #step_width = 80 / @bbrc.GetNoRootNodes().to_f
+
+        $logger.debug "BBRC setup: #{Time.now-time}"
+        time = Time.now
+        ftime = 0
+        itime = 0
+        rtime = 0
+  
+        # run @bbrc
+        (0 .. @bbrc.GetNoRootNodes()-1).each do |j|
+          results = @bbrc.MineRoot(j)
+          results.each do |result|
+            rt = Time.now
+            f = YAML.load(result)[0]
+            smarts = f.shift
+            # convert fminer SMARTS representation into a more human readable format
+            smarts.gsub!(%r{\[#(\d+)&(\w)\]}) do
+             element = TABLE_OF_ELEMENTS[$1.to_i-1]
+             $2 == "a" ? element.downcase : element
+            end
+            p_value = f.shift
+            f.flatten!
+            compound_idxs = f.collect{|e| e.first.first-1}
+            # majority class
+            effect = compound_idxs.collect{|i| training_dataset.data_entries[i].first}.mode
+  
+=begin
+            if (!@bbrc.GetRegression)
+              id_arrs = f[2..-1].flatten
+              max = OpenTox::Algorithm::Fminer.effect(f[2..-1].reverse, @fminer.db_class_sizes) # f needs reversal for bbrc
+              effect = max+1
+            else #regression part
+              id_arrs = f[2]
+              # DV: effect calculation
+              f_arr=Array.new
+              f[2].each do |id|
+                id=id.keys[0] # extract id from hit count hash
+                f_arr.push(@fminer.all_activities[id])
+              end
+              f_median=f_arr.to_scale.median
+              if g_median >= f_median
+                effect = 'activating'
+              else
+                effect = 'deactivating'
+              end
+            end
+=end
+            rtime += Time.now - rt
+  
+            ft = Time.now
+            feature = OpenTox::FminerSmarts.find_or_create_by({
+              "smarts" => smarts,
+              "p_value" => p_value.to_f.abs.round(5),
+              "effect" => effect,
+              "dataset_id" => feature_dataset.id
+            })
+            feature_dataset.feature_ids << feature.id
+            ftime += Time.now - ft
+
+            it = Time.now
+            f.each do |id_count_hash|
+              id_count_hash.each do |id,count|
+                nr_hits ? count = count.to_i : count = 1
+                feature_dataset.data_entries[id-1] ||= []
+                feature_dataset.data_entries[id-1][feature_dataset.feature_ids.size-1] = count
+              end
+            end
+            itime += Time.now - it
+  
+          end
+        end
+
+        $logger.debug "Fminer: #{Time.now-time} (read: #{rtime}, iterate: #{itime}, find/create Features: #{ftime})"
+        time = Time.now
+
+        feature_dataset.fill_nil_with 0
+
+        $logger.debug "Prepare save: #{Time.now-time}"
+        time = Time.now
+        feature_dataset.save_all
+
+        $logger.debug "Save: #{Time.now-time}"
+        feature_dataset
+  
+      end
+    end
+  end
+end

data/lib/classification.rb

@@ -0,0 +1,107 @@
+module OpenTox
+  module Algorithm
+    
+    class Classification
+
+      def self.weighted_majority_vote neighbors
+        return {:value => nil,:confidence => nil,:warning => "Cound not find similar compounds."} if neighbors.empty?
+        weighted_sum = {}
+        sim_sum = 0.0
+        neighbors.each do |row|
+          n,sim,acts = row
+          acts.each do |act|
+            weighted_sum[act] ||= 0
+            weighted_sum[act] += sim
+          end
+        end
+        case weighted_sum.size
+        when 1
+          return {:value => weighted_sum.keys.first, :confidence => weighted_sum.values.first/neighbors.size.abs}
+        when 2
+          sim_sum = weighted_sum[weighted_sum.keys[0]]
+          sim_sum -= weighted_sum[weighted_sum.keys[1]]
+          sim_sum > 0 ? prediction = weighted_sum.keys[0] : prediction = weighted_sum.keys[1] 
+          confidence = (sim_sum/neighbors.size).abs 
+          return {:value => prediction,:confidence => confidence}
+        else
+          bad_request_error "Cannot predict more than 2 classes, multinomial classifications is not yet implemented. Received classes were: '#{weighted.sum.keys}'"
+        end
+      end
+
+      # Classification with majority vote from neighbors weighted by similarity
+      # @param [Hash] params Keys `:activities, :sims, :value_map` are required
+      # @return [Numeric] A prediction value.
+      def self.fminer_weighted_majority_vote neighbors, training_dataset
+
+        neighbor_contribution = 0.0
+        confidence_sum = 0.0
+
+        $logger.debug "Weighted Majority Vote Classification."
+
+        values = neighbors.collect{|n| n[2]}.uniq
+        neighbors.each do |neighbor|
+          i = training_dataset.compound_ids.index n.id
+          neighbor_weight = neighbor[1]
+          activity = values.index(neighbor[2]) + 1 # map values to integers > 1
+          neighbor_contribution += activity * neighbor_weight
+          if values.size == 2 # AM: provide compat to binary classification: 1=>false 2=>true
+            case activity
+            when 1
+              confidence_sum -= neighbor_weight
+            when 2
+              confidence_sum += neighbor_weight
+            end
+          else
+            confidence_sum += neighbor_weight
+          end
+        end
+        if values.size == 2 
+          if confidence_sum >= 0.0
+            prediction = values[1]
+          elsif confidence_sum < 0.0
+            prediction = values[0] 
+          end
+        elsif values.size == 1 # all neighbors have the same value
+          prediction = values[0] 
+        else 
+          prediction = (neighbor_contribution/confidence_sum).round  # AM: new multinomial prediction
+        end 
+
+        confidence = (confidence_sum/neighbors.size).abs 
+        {:value => prediction, :confidence => confidence.abs}
+      end
+
+      # Local support vector regression from neighbors 
+      # @param [Hash] params Keys `:props, :activities, :sims, :min_train_performance` are required
+      # @return [Numeric] A prediction value.
+      def self.local_svm_classification(params)
+
+        confidence = 0.0
+        prediction = nil
+
+        $logger.debug "Local SVM."
+        if params[:activities].size>0
+          if params[:props]
+            n_prop = params[:props][0].collect.to_a
+            q_prop = params[:props][1].collect.to_a
+            props = [ n_prop, q_prop ]
+          end
+          activities = params[:activities].collect.to_a
+          activities = activities.collect{|v| "Val" + v.to_s} # Convert to string for R to recognize classification
+          prediction = local_svm_prop( props, activities, params[:min_train_performance]) # params[:props].nil? signals non-prop setting
+          prediction = prediction.sub(/Val/,"") if prediction # Convert back
+          confidence = 0.0 if prediction.nil?
+          #$logger.debug "Prediction: '" + prediction.to_s + "' ('#{prediction.class}')."
+          confidence = get_confidence({:sims => params[:sims][1], :activities => params[:activities]})
+        end
+        {:value => prediction, :confidence => confidence}
+
+      end
+
+
+
+    end
+
+  end
+end
+

data/lib/compound.rb

@@ -0,0 +1,254 @@
+# TODO: check
+# *** Open Babel Error  in ParseFile
+#    Could not find contribution data file.
+
+CACTUS_URI="http://cactus.nci.nih.gov/chemical/structure/"
+
+module OpenTox
+
+  class Compound
+    include OpenTox
+
+    field :inchi, type: String
+    field :smiles, type: String
+    field :inchikey, type: String
+    field :names, type: Array
+    field :warning, type: String
+    field :cid, type: String
+    field :chemblid, type: String
+    field :png_id, type: BSON::ObjectId
+    field :svg_id, type: BSON::ObjectId
+    field :sdf_id, type: BSON::ObjectId
+    field :fp4, type: Array
+    field :fp4_size, type: Integer
+
+    # Overwrites standard Mongoid method to create fingerprints before database insertion
+    def self.find_or_create_by params
+      compound = self.find_or_initialize_by params
+      unless compound.fp4 and !compound.fp4.empty?
+        compound.fp4_size = 0
+        compound.fp4 = []
+        fingerprint = FingerprintSmarts.fingerprint
+        Algorithm::Descriptor.smarts_match(compound, fingerprint).each_with_index do |m,i|
+          if m > 0
+            compound.fp4 << fingerprint[i].id
+            compound.fp4_size += 1
+          end
+        end
+      end
+      compound.save
+      compound
+    end
+
+    # Create a compound from smiles string
+    # @example
+    #   compound = OpenTox::Compound.from_smiles("c1ccccc1")
+    # @param [String] smiles Smiles string
+    # @return [OpenTox::Compound] Compound
+    def self.from_smiles smiles
+      smiles = obconversion(smiles,"smi","can")
+      if smiles.empty?
+        Compound.find_or_create_by(:warning => "SMILES parsing failed for '#{smiles}', this may be caused by an incorrect SMILES string.")
+      else
+        Compound.find_or_create_by :smiles => obconversion(smiles,"smi","can")
+      end
+    end
+
+    # Create a compound from inchi string
+    # @param inchi [String] smiles InChI string
+    # @return [OpenTox::Compound] Compound
+    def self.from_inchi inchi
+      # Temporary workaround for OpenBabels Inchi bug
+      # http://sourceforge.net/p/openbabel/bugs/957/
+      # bug has not been fixed in latest git/development version
+      #smiles = `echo "#{inchi}" | "#{File.join(File.dirname(__FILE__),"..","openbabel","bin","babel")}" -iinchi - -ocan`.chomp.strip
+      smiles = obconversion(inchi,"inchi","can")
+      if smiles.empty?
+        Compound.find_or_create_by(:warning => "InChi parsing failed for #{inchi}, this may be caused by an incorrect InChi string or a bug in OpenBabel libraries.")
+      else
+        Compound.find_or_create_by(:smiles => smiles, :inchi => inchi)
+      end
+    end
+
+    # Create a compound from sdf string
+    # @param sdf [String] smiles SDF string
+    # @return [OpenTox::Compound] Compound
+    def self.from_sdf sdf
+      # do not store sdf because it might be 2D
+      Compound.from_smiles obconversion(sdf,"sdf","can")
+    end
+
+    # Create a compound from name. Relies on an external service for name lookups.
+    # @example
+    #   compound = OpenTox::Compound.from_name("Benzene")
+    # @param name [String] can be also an InChI/InChiKey, CAS number, etc
+    # @return [OpenTox::Compound] Compound
+    def self.from_name name
+      Compound.from_smiles RestClientWrapper.get(File.join(CACTUS_URI,URI.escape(name),"smiles"))
+    end
+
+    # Get InChI
+    # @return [String] InChI string
+    def inchi
+      unless self["inchi"]
+
+        result = obconversion(smiles,"smi","inchi")
+        #result = `echo "#{self.smiles}" | "#{File.join(File.dirname(__FILE__),"..","openbabel","bin","babel")}" -ismi - -oinchi`.chomp
+        update(:inchi => result.chomp) unless result.empty?
+      end
+      self["inchi"]
+    end
+
+    # Get InChIKey
+    # @return [String] InChIKey string
+    def inchikey
+      update(:inchikey => obconversion(smiles,"smi","inchikey")) unless self["inchikey"]
+      self["inchikey"]
+    end
+
+    # Get (canonical) smiles
+    # @return [String] Smiles string
+    def smiles
+      update(:smiles => obconversion(self["smiles"],"smi","can")) unless self["smiles"] 
+      self["smiles"]
+    end
+
+    # Get sdf
+    # @return [String] SDF string
+    def sdf
+      if self.sdf_id.nil? 
+        sdf = obconversion(smiles,"smi","sdf")
+        file = Mongo::Grid::File.new(sdf, :filename => "#{id}.sdf",:content_type => "chemical/x-mdl-sdfile")
+        sdf_id = $gridfs.insert_one file
+        update :sdf_id => sdf_id
+      end
+      $gridfs.find_one(_id: self.sdf_id).data
+    end
+
+    # Get SVG image
+    # @return [image/svg] Image data
+    def svg
+      if self.svg_id.nil?
+       svg = obconversion(smiles,"smi","svg")
+       file = Mongo::Grid::File.new(svg, :filename => "#{id}.svg", :content_type => "image/svg")
+       update(:image_id => $gridfs.insert_one(file))
+      end
+      $gridfs.find_one(_id: self.svg_id).data
+
+    end
+
+    # Get png image
+    # @example
+    #   image = compound.png
+    # @return [image/png] Image data
+    def png
+      if self.png_id.nil?
+       png = obconversion(smiles,"smi","_png2")
+       file = Mongo::Grid::File.new(Base64.encode64(png), :filename => "#{id}.png", :content_type => "image/png")
+       update(:png_id => $gridfs.insert_one(file))
+      end
+      Base64.decode64($gridfs.find_one(_id: self.png_id).data)
+
+    end
+
+    # Get all known compound names. Relies on an external service for name lookups.
+    # @example
+    #   names = compound.names
+    # @return [String] Compound names
+    def names
+      update(:names => RestClientWrapper.get("#{CACTUS_URI}#{inchi}/names").split("\n")) unless self["names"] 
+      self["names"]
+    end
+
+    # @return [String] PubChem Compound Identifier (CID), derieved via restcall to pubchem
+    def cid
+      pug_uri = "http://pubchem.ncbi.nlm.nih.gov/rest/pug/"
+      update(:cid => RestClientWrapper.post(File.join(pug_uri, "compound", "inchi", "cids", "TXT"),{:inchi => inchi}).strip) unless self["cid"] 
+      self["cid"]
+    end
+
+    # @return [String] ChEMBL database compound id, derieved via restcall to chembl
+    def chemblid
+      # https://www.ebi.ac.uk/chembldb/ws#individualCompoundByInChiKey
+      uri = "http://www.ebi.ac.uk/chemblws/compounds/smiles/#{smiles}.json"
+      update(:chemblid => JSON.parse(RestClientWrapper.get(uri))["compounds"].first["chemblId"]) unless self["chemblid"] 
+      self["chemblid"]
+    end
+
+    def neighbors threshold=0.7
+      # from http://blog.matt-swain.com/post/87093745652/chemical-similarity-search-in-mongodb
+      qn = fp4.size
+      #qmin = qn * threshold
+      #qmax = qn / threshold
+      #not sure if it is worth the effort of keeping feature counts up to date (compound deletions, additions, ...)
+      #reqbits = [count['_id'] for count in db.mfp_counts.find({'_id': {'$in': qfp}}).sort('count', 1).limit(qn - qmin + 1)]
+      aggregate = [
+        #{'$match': {'mfp.count': {'$gte': qmin, '$lte': qmax}, 'mfp.bits': {'$in': reqbits}}},
+        {'$match' =>  {'_id' => {'$ne' => self.id}}}, # remove self
+        {'$project' => {
+          'tanimoto' => {'$let' => {
+            'vars' => {'common' => {'$size' => {'$setIntersection' => ['$fp4', fp4]}}},
+            'in' => {'$divide' => ['$$common', {'$subtract' => [{'$add' => [qn, '$fp4_size']}, '$$common']}]}
+          }},
+          '_id' => 1
+        }},
+        {'$match' =>  {'tanimoto' => {'$gte' => threshold}}},
+        {'$sort' => {'tanimoto' => -1}}
+      ]
+      
+      $mongo["compounds"].aggregate(aggregate).collect{ |r| [r["_id"], r["tanimoto"]] }
+        
+    end
+=begin
+=end
+
+    private
+
+    def self.obconversion(identifier,input_format,output_format,option=nil)
+      obconversion = OpenBabel::OBConversion.new
+      obconversion.set_options(option, OpenBabel::OBConversion::OUTOPTIONS) if option
+      obmol = OpenBabel::OBMol.new
+      obconversion.set_in_and_out_formats input_format, output_format
+      obconversion.read_string obmol, identifier
+      case output_format
+      when /smi|can|inchi/
+        obconversion.write_string(obmol).gsub(/\s/,'').chomp
+      when /sdf/
+p "SDF conversion"
+        # has no effect
+	#obconversion.add_option("gen3D", OpenBabel::OBConversion::GENOPTIONS)
+        # segfaults with openbabel git master 
+        #OpenBabel::OBOp.find_type("Gen3D").do(obmol) 
+
+        # TODO: find disconnected structures
+        # strip_salts
+        # separate
+        obmol.add_hydrogens
+        builder = OpenBabel::OBBuilder.new
+        builder.build(obmol)
+
+        sdf = obconversion.write_string(obmol)
+print sdf
+        if sdf.match(/.nan/)
+          
+# TODO: fix or eliminate 2d generation
+          $logger.warn "3D generation failed for compound #{identifier}, trying to calculate 2D structure"
+          obconversion.set_options("gen2D", OpenBabel::OBConversion::GENOPTIONS)
+          #OpenBabel::OBOp.find_type("Gen2D").do(obmol) 
+          sdf = obconversion.write_string(obmol)
+          if sdf.match(/.nan/)
+            $logger.warn "2D generation failed for compound #{identifier}"
+            sdf = nil
+          end
+        end
+        sdf
+      else
+        obconversion.write_string(obmol)
+      end
+    end
+
+    def obconversion(identifier,input_format,output_format,option=nil)
+      self.class.obconversion(identifier,input_format,output_format,option=nil)
+    end
+  end
+end