opentox-ruby 3.0.1 → 3.1.0

data/lib/authorization.rb

@@ -37,13 +37,15 @@ module OpenTox
       
       #Loads and sends Policyfile(XML) to open-sso server
       # @param [String] URI to create a policy for      
-      def send(uri)    
+      def send(uri)
         xml = get_xml(uri)
         ret = false
-        ret = Authorization.create_policy(xml, @subjectid) 
+        ret = Authorization.create_policy(xml, @subjectid)
+        LOGGER.warn "Create policy on openSSO failed for URI: #{uri} subjectid: #{@subjectid}. Will try again." if !ret
+        ret = Authorization.create_policy(xml, @subjectid) if !ret
         LOGGER.debug "Policy send with subjectid: #{@subjectid}"
         LOGGER.warn "Not created Policy is: #{xml}" if !ret
-        ret  
+        ret
       end
       
     end
@@ -337,7 +339,7 @@ module OpenTox
     # @param [String] subjectid
     # @return [Boolean] true if access granted, else otherwise
     def self.authorized?(uri, request_method, subjectid)
-      if CONFIG[:authorization][:free_request].include?(request_method)  
+      if CONFIG[:authorization][:free_request].include?(request_method)
         #LOGGER.debug "authorized? >>true<< (request is free), method: #{request_method}, URI: #{uri}, subjectid: #{subjectid}"
         true
       elsif OpenTox::Authorization.free_uri?(uri, request_method)
@@ -360,7 +362,7 @@ module OpenTox
         false
       end
     end
-    
+
     private
     def self.free_uri?(uri, request_method)
       if CONFIG[:authorization][:free_uris]
@@ -374,7 +376,7 @@ module OpenTox
       end    
       return false
     end
-    
+
     def self.authorize_exception?(uri, request_method)
       if CONFIG[:authorization][:authorize_exceptions]
         CONFIG[:authorization][:authorize_exceptions].each do |request_methods,uris|
@@ -387,6 +389,6 @@ module OpenTox
       end    
       return false
     end    
-    
+
   end
-end
+end

data/lib/compound.rb

@@ -6,13 +6,15 @@ module OpenTox
   # Ruby wrapper for OpenTox Compound Webservices (http://opentox.org/dev/apis/api-1.2/structure).
 	class Compound 
 
+    include OpenTox
+
 		attr_accessor :inchi, :uri
 
 		# Create compound with optional uri
     # @example
-    #   compound = OpenTox::Compound.new("http://webservices.in-silico.ch/compound/InChI=1S/C6H6/c1-2-4-6-5-3-1/h1-6H"")
+    #   compound = Compound.new("http://webservices.in-silico.ch/compound/InChI=1S/C6H6/c1-2-4-6-5-3-1/h1-6H"")
     # @param [optional, String] uri Compound URI
-    # @return [OpenTox::Compound] Compound
+    # @return [Compound] Compound
 		def initialize(uri=nil)
       @uri = uri
       case @uri
@@ -36,9 +38,9 @@ module OpenTox
 
     # Create a compound from smiles string
     # @example
-    #   compound = OpenTox::Compound.from_smiles("c1ccccc1")
+    #   compound = Compound.from_smiles("c1ccccc1")
     # @param [String] smiles Smiles string
-    # @return [OpenTox::Compound] Compound
+    # @return [Compound] Compound
     def self.from_smiles(smiles)
       c = Compound.new
       c.inchi = Compound.smiles2inchi(smiles)
@@ -48,7 +50,7 @@ module OpenTox
 
     # Create a compound from inchi string
     # @param [String] smiles InChI string
-    # @return [OpenTox::Compound] Compound
+    # @return [Compound] Compound
     def self.from_inchi(inchi)
       c = Compound.new
       c.inchi = inchi
@@ -58,7 +60,7 @@ module OpenTox
 
     # Create a compound from sdf string
     # @param [String] smiles SDF string
-    # @return [OpenTox::Compound] Compound
+    # @return [Compound] Compound
     def self.from_sdf(sdf)
       c = Compound.new
       c.inchi = Compound.sdf2inchi(sdf)
@@ -68,9 +70,9 @@ module OpenTox
 
     # Create a compound from name. Relies on an external service for name lookups.
     # @example
-    #   compound = OpenTox::Compound.from_name("Benzene")
+    #   compound = Compound.from_name("Benzene")
     # @param [String] name name can be also an InChI/InChiKey, CAS number, etc
-    # @return [OpenTox::Compound] Compound
+    # @return [Compound] Compound
     def self.from_name(name)
       c = Compound.new
       # paranoid URI encoding to keep SMILES charges and brackets
@@ -131,7 +133,7 @@ module OpenTox
 
 		# Match a smarts string
     # @example
-    #   compound = OpenTox::Compound.from_name("Benzene")
+    #   compound = Compound.from_name("Benzene")
     #   compound.match?("cN") # returns false
     # @param [String] smarts Smarts string
 		def match?(smarts)
@@ -146,7 +148,7 @@ module OpenTox
 
 		# Match an array of smarts strings, returns array with matching smarts
     # @example
-    #   compound = OpenTox::Compound.from_name("Benzene")
+    #   compound = Compound.from_name("Benzene")
     #   compound.match(['cc','cN']) # returns ['cc']
     # @param [Array] smarts_array Array with Smarts strings
     # @return [Array] Array with matching Smarts strings
@@ -166,7 +168,7 @@ module OpenTox
 
     # Match_hits an array of smarts strings, returns hash with matching smarts as key and number of non-unique hits as value 
     # @example
-    #   compound = OpenTox::Compound.from_name("Benzene")
+    #   compound = Compound.from_name("Benzene")
     #   compound.match(['cc','cN']) # returns ['cc']
     # @param [Array] smarts_array Array with Smarts strings
     # @return [Hash] Hash with matching smarts as key and number of non-unique hits as value
@@ -191,6 +193,40 @@ module OpenTox
       return smarts_hits
       #smarts_array.collect { |s| s if match?(s)}.compact
 		end
+    
+    # Lookup numerical values, returns hash with feature name as key and value as value 
+    # @param [Array] Array of feature names
+    # @param [String] Feature dataset uri
+    # @return [Hash] Hash with feature name as key and value as value
+    def lookup(feature_array,feature_dataset_uri,pc_type,subjectid=nil)
+      ds = OpenTox::Dataset.find(feature_dataset_uri,subjectid)
+      #entry = ds.data_entries[self.uri]
+      entry = nil 
+      ds.data_entries.each { |c_uri, values| 
+        if c_uri.split('/compound/').last == self.to_inchi
+          entry = ds.data_entries[self.uri]
+          break
+        end
+      }
+      LOGGER.debug "#{entry.size} entries in feature ds for query." unless entry.nil?
+
+      if entry.nil?
+        uri, smiles_to_inchi = OpenTox::Algorithm.get_pc_descriptors({:compounds => [self.uri], :pc_type => pc_type})
+        uri = OpenTox::Algorithm.load_ds_csv(uri, smiles_to_inchi, subjectid)
+        ds = OpenTox::Dataset.find(uri,subjectid)
+        entry = ds.data_entries[self.uri]
+        ds.delete(subjectid)
+      end
+      features = entry.keys
+      features.each { |feature| 
+        new_feature = File.join(feature_dataset_uri, "feature", feature.split("/").last) 
+        entry[new_feature] = entry[feature].flatten.first.to_f # see algorithm/lazar.rb:182, to_f because feature type detection doesn't work w 1 entry
+        entry.delete(feature) unless feature == new_feature # e.g. when loading from ambit
+      }
+      #res = feature_array.collect {|v| entry[v]}
+      #LOGGER.debug "----- am #{entry.to_yaml}"
+      entry
+		end
 
 
     # Get URI of compound image with highlighted fragments

data/lib/dataset.rb

@@ -288,7 +288,7 @@ module OpenTox
 
     # Insert a statement (compound_uri,feature_uri,value)
     # @example Insert a statement (compound_uri,feature_uri,value)
-    #   dataset.add "http://webservices.in-silico.ch/compound/InChI=1S/C6Cl6/c7-1-2(8)4(10)6(12)5(11)3(1)9", "http://webservices.in-silico.ch/dataset/1/feature/hamster_carcinogenicity", true
+    #   dataset.add "http://webservices.in-silico.ch/compound/InChI=1S/C6Cl6/c7-1-2(8)4(10)6(12)5(11)3(1)9", "http://webservices.in-silico.ch/dataset/1/feature/hamster_carcinogenicity", 1
     # @param [String] compound Compound URI
     # @param [String] feature Compound URI
     # @param [Boolean,Float] value Feature value
@@ -315,6 +315,16 @@ module OpenTox
       @features[feature] = metadata
     end
 
+    # Complete feature values by adding zeroes
+    def complete_data_entries
+      all_features = @features.keys
+      @data_entries.each { |c, e|
+        (Set.new(all_features.collect)).subtract(Set.new e.keys).to_a.each { |f|
+          self.add(c,f,0)
+        }
+      }
+    end
+
     # Add/modify metadata for a feature
     # @param [String] feature Feature URI
     # @param [Hash] metadata Hash with feature metadata
@@ -363,7 +373,45 @@ module OpenTox
       dataset.save(subjectid)
       dataset
     end
-
+    
+    # merges two dataset into a new dataset (by default uses all compounds and features)
+    # precondition: both datasets are fully loaded
+    # @param [OpenTox::Dataset] dataset1 to merge
+    # @param [OpenTox::Dataset] dataset2 to merge
+    # @param [Hash] metadata
+    # @param [optional,String] subjectid
+    # @param [optional,Array] features1, if specified only this features of dataset1 are used
+    # @param [optional,Array] features2, if specified only this features of dataset2 are used
+    # @param [optional,Array] compounds1, if specified only this compounds of dataset1 are used
+    # @param [optional,Array] compounds2, if specified only this compounds of dataset2 are used
+    # example: if you want no features from dataset2, give empty array as features2
+    def self.merge( dataset1, dataset2, metadata, subjectid=nil, features1=nil, features2=nil, compounds1=nil, compounds2=nil )
+      features1 = dataset1.features.keys unless features1
+      features2 = dataset2.features.keys unless features2
+      compounds1 = dataset1.compounds unless compounds1
+      compounds2 = dataset2.compounds unless compounds2
+      data_combined = OpenTox::Dataset.create(CONFIG[:services]["opentox-dataset"],subjectid)
+      LOGGER.debug("merging datasets #{dataset1.uri} and #{dataset2.uri} to #{data_combined.uri}")
+      [[dataset1, features1, compounds1], [dataset2, features2, compounds2]].each do |dataset,features,compounds|
+        compounds.each{|c| data_combined.add_compound(c)}
+        features.each do |f|
+          m = dataset.features[f]
+          m[OT.hasSource] = dataset.uri unless m[OT.hasSource]
+          data_combined.add_feature(f,m)
+          compounds.each do |c|
+            dataset.data_entries[c][f].each do |v|
+              data_combined.add(c,f,v)
+            end if dataset.data_entries[c] and dataset.data_entries[c][f]
+          end
+        end
+      end
+      metadata = {} unless metadata
+      metadata[OT.hasSource] = "Merge from #{dataset1.uri} and #{dataset2.uri}" unless metadata[OT.hasSource]
+      data_combined.add_metadata(metadata)
+      data_combined.save(subjectid)
+      data_combined
+    end
+    
     # Save dataset at the dataset service 
     # - creates a new dataset if uri is not set
     # - overwrites dataset if uri exists

data/lib/environment.rb

@@ -24,7 +24,11 @@ end
 
 # database
 #`redis-server /opt/redis/redis.conf` unless File.exists? "/var/run/redis.pid" # removed by AM
-Ohm.connect :thread_safe => true
+ohm_port=6379
+if !CONFIG[:ohm_port].nil? 
+  ohm_port=CONFIG[:ohm_port].to_i
+end
+Ohm.connect(:thread_safe => true, :port => ohm_port)
 
 # load mail settings for error messages
 #load File.join config_dir,"mail.rb" if File.exists?(File.join config_dir,"mail.rb")
@@ -87,4 +91,5 @@ DC =  OwlNamespace.new 'http://purl.org/dc/elements/1.1/'
 OT =  OwlNamespace.new 'http://www.opentox.org/api/1.1#'
 OTA =  OwlNamespace.new 'http://www.opentox.org/algorithmTypes.owl#'
 XSD = OwlNamespace.new 'http://www.w3.org/2001/XMLSchema#'
+#BO = OwlNamespace.new 'http://www.blueobelisk.org/ontologies/chemoinformatics-algorithms/#'
 

data/lib/model.rb

@@ -102,8 +102,8 @@ module OpenTox
       include Algorithm
       include Model
 
-      attr_accessor :compound, :prediction_dataset, :features, :effects, :activities, :p_values, :fingerprints, :feature_calculation_algorithm, :similarity_algorithm, :prediction_algorithm, :min_sim, :subjectid, :prop_kernel, :value_map, :nr_hits, :transform, :conf_stdev, :prediction_min_max
 
+      attr_accessor :compound, :prediction_dataset, :features, :effects, :activities, :p_values, :fingerprints, :feature_calculation_algorithm, :similarity_algorithm, :prediction_algorithm, :subjectid, :value_map, :compound_fingerprints, :feature_calculation_algorithm, :neighbors
       def initialize(uri=nil)
 
         if uri
@@ -120,18 +120,11 @@ module OpenTox
         @p_values = {}
         @fingerprints = {}
         @value_map = {}
-        @prediction_min_max = [] 
 
         @feature_calculation_algorithm = "Substructure.match"
         @similarity_algorithm = "Similarity.tanimoto"
         @prediction_algorithm = "Neighbors.weighted_majority_vote"
         
-        @nr_hits = false
-        @min_sim = 0.3
-        @prop_kernel = false
-        @transform = { "class" => "NOP"  }
-        @conf_stdev = false
-
       end
 
       # Get URIs of all lazar models
@@ -174,19 +167,14 @@ module OpenTox
         lazar.feature_calculation_algorithm = hash["feature_calculation_algorithm"] if hash["feature_calculation_algorithm"]
         lazar.similarity_algorithm = hash["similarity_algorithm"] if hash["similarity_algorithm"]
         lazar.prediction_algorithm = hash["prediction_algorithm"] if hash["prediction_algorithm"]
-        lazar.min_sim = hash["min_sim"] if hash["min_sim"]
         lazar.subjectid = hash["subjectid"] if hash["subjectid"]
-        lazar.prop_kernel = hash["prop_kernel"] if hash["prop_kernel"]
         lazar.value_map = hash["value_map"] if hash["value_map"]
-        lazar.nr_hits = hash["nr_hits"] if hash["nr_hits"]
-        lazar.transform = hash["transform"] if hash["transform"]
-        lazar.conf_stdev = hash["conf_stdev"] if hash["conf_stdev"]
-        lazar.prediction_min_max = hash["prediction_min_max"] if hash["prediction_min_max"]
+
         lazar
       end
 
       def to_json
-        Yajl::Encoder.encode({:uri => @uri,:metadata => @metadata, :compound => @compound, :prediction_dataset => @prediction_dataset, :features => @features, :effects => @effects, :activities => @activities, :p_values => @p_values, :fingerprints => @fingerprints, :feature_calculation_algorithm => @feature_calculation_algorithm, :similarity_algorithm => @similarity_algorithm, :prediction_algorithm => @prediction_algorithm, :min_sim => @min_sim, :subjectid => @subjectid, :prop_kernel => @prop_kernel, :value_map => @value_map, :nr_hits => @nr_hits, :transform => @transform, :conf_stdev => @conf_stdev, :prediction_min_max => @prediction_min_max})
+        Yajl::Encoder.encode({:uri => @uri,:metadata => @metadata, :compound => @compound, :prediction_dataset => @prediction_dataset, :features => @features, :effects => @effects, :activities => @activities, :p_values => @p_values, :fingerprints => @fingerprints, :feature_calculation_algorithm => @feature_calculation_algorithm, :similarity_algorithm => @similarity_algorithm, :prediction_algorithm => @prediction_algorithm, :subjectid => @subjectid, :value_map => @value_map})
       end
 
       def run( params, accept_header=nil, waiting_task=nil )
@@ -230,8 +218,11 @@ module OpenTox
             predict(compound_uri,false,subjectid)
             count += 1
             waiting_task.progress( count/d.compounds.size.to_f*100.0 ) if waiting_task
-          rescue => ex
-            LOGGER.warn "prediction for compound "+compound_uri.to_s+" failed: "+ex.message+" subjectid: #{subjectid}"
+          rescue => e
+            LOGGER.warn "prediction for compound "+compound_uri.to_s+" failed: "+e.message+" subjectid: #{subjectid}"
+            #LOGGER.debug "#{e.class}: #{e.message}"
+            #LOGGER.debug "Backtrace:\n\t#{e.backtrace.join("\n\t")}"
+
           end
         end
         #@prediction_dataset.save(subjectid)
@@ -246,7 +237,6 @@ module OpenTox
 
         @compound = Compound.new compound_uri
         features = {}
-
         #LOGGER.debug self.to_yaml
         unless @prediction_dataset
           @prediction_dataset = Dataset.create(CONFIG[:services]["opentox-dataset"], subjectid)
@@ -257,29 +247,42 @@ module OpenTox
             OT.parameters => [{DC.title => "compound_uri", OT.paramValue => compound_uri}]
           } )
         end
-
         if OpenTox::Feature.find(metadata[OT.dependentVariables], subjectid).feature_type == "regression"
           all_activities = [] 
           all_activities = @activities.values.flatten.collect! { |i| i.to_f }
-          @prediction_min_max[0] = (all_activities.to_scale.min/2)
-          @prediction_min_max[1] = (all_activities.to_scale.max*2)
         end
-
         unless database_activity(subjectid) # adds database activity to @prediction_dataset
+          # Calculation of needed values for query compound
+          @compound_features = eval("#{@feature_calculation_algorithm}({
+                                    :compound => @compound, 
+                                    :features => @features, 
+                                    :feature_dataset_uri => @metadata[OT.featureDataset],
+                                    :pc_type => self.parameter(\"pc_type\"),
+                                    :subjectid => subjectid
+                                    })")
+          # Adding fingerprint of query compound with features and values(p_value*nr_hits)
+          @compound_fingerprints = {}
+          @compound_features.each do |feature, value| # value is nil if "Substructure.match"
+            if @feature_calculation_algorithm == "Substructure.match_hits" 
+              @compound_fingerprints[feature] = @p_values[feature] * value
+            elsif @feature_calculation_algorithm == "Substructure.match"
+              @compound_fingerprints[feature] = @p_values[feature]
+            elsif @feature_calculation_algorithm == "Substructure.lookup"
+              @compound_fingerprints[feature] = value
+            end
+          end
 
-          neighbors
-          prediction = eval("#{@prediction_algorithm} ( { :neighbors => @neighbors, 
-                                                          :compound => @compound,
-                                                          :features => @features, 
-                                                          :p_values => @p_values, 
-                                                          :fingerprints => @fingerprints,
-                                                          :similarity_algorithm => @similarity_algorithm, 
-                                                          :prop_kernel => @prop_kernel,
+          # Transform model data to machine learning scheme (tables of data)
+          mtf = OpenTox::Algorithm::Transform::ModelTransformer.new(self)
+          mtf.transform
+
+          # Make a prediction
+          prediction = eval("#{@prediction_algorithm}( { :props => mtf.props,
+                                                          :acts => mtf.acts,
+                                                          :sims => mtf.sims,
                                                           :value_map => @value_map,
-                                                          :nr_hits => @nr_hits,
-                                                          :conf_stdev => @conf_stdev,
-                                                          :prediction_min_max => @prediction_min_max,
-                                                          :transform => @transform } ) ")
+                                                          :min_train_performance => self.parameter(\"min_train_performance\")
+                                                        } ) ")
 
           value_feature_uri = File.join( @uri, "predicted", "value")
           confidence_feature_uri = File.join( @uri, "predicted", "confidence")
@@ -355,44 +358,6 @@ module OpenTox
         @prediction_dataset
       end
 
-      
-
-      # Find neighbors and store them as object variable, access all compounds for that.
-      def neighbors
-        @compound_features = eval("#{@feature_calculation_algorithm}(@compound,@features)") if @feature_calculation_algorithm
-        @neighbors = []
-        @fingerprints.keys.each do |training_compound| # AM: access all compounds
-          add_neighbor @fingerprints[training_compound].keys, training_compound
-        end
-      end
-
-      # Adds a neighbor to @neighbors if it passes the similarity threshold.
-      def add_neighbor(training_features, training_compound)
-        compound_features_hits = {}
-        training_compound_features_hits = {}
-        if @nr_hits
-          compound_features_hits = @compound.match_hits(@compound_features)
-          training_compound_features_hits = @fingerprints[training_compound]
-          #LOGGER.debug "dv ------------ training_compound_features_hits:#{training_compound_features_hits.class}  #{training_compound_features_hits}"
-        end
-        params = {}
-        params[:nr_hits] = @nr_hits
-        params[:compound_features_hits] = compound_features_hits
-        params[:training_compound_features_hits] = training_compound_features_hits
-
-        sim = eval("#{@similarity_algorithm}(training_features, @compound_features, @p_values, params)")
-        if sim > @min_sim
-          @activities[training_compound].each do |act|
-            @neighbors << {
-              :compound => training_compound,
-              :similarity => sim,
-              :features => training_features,
-              :activity => act
-            }
-          end
-        end
-      end
-
       # Find database activities and store them in @prediction_dataset
       # @return [Boolean] true if compound has databasse activities, false if not
       def database_activity(subjectid)

data/lib/opentox-ruby.rb

@@ -9,6 +9,6 @@ rescue LoadError
 end
 
 ['opentox', 'compound','dataset', 'parser','serializer', 'algorithm','model','task','validation','feature', 
- 'rest_client_wrapper', 'authorization', 'policy', 'helper', 'to-html', 'ontology' ].each do |lib|
+ 'rest_client_wrapper', 'authorization', 'policy', 'helper', 'to-html', 'ontology', 'r-util' ].each do |lib|
 	require lib
 end