lazar 0.0.7 → 0.0.9

data/lib/leave-one-out-validation.rb

@@ -0,0 +1,205 @@
+module OpenTox
+
+  class LeaveOneOutValidation
+
+    field :model_id, type: BSON::ObjectId
+    field :dataset_id, type: BSON::ObjectId
+    field :nr_instances, type: Integer
+    field :nr_unpredicted, type: Integer
+    field :predictions, type: Array
+    field :finished_at, type: Time 
+
+    def self.create model
+      model.training_dataset.features.first.nominal? ? klass = ClassificationLeaveOneOutValidation : klass = RegressionLeaveOneOutValidation
+      loo = klass.new :model_id => model.id, :dataset_id => model.training_dataset_id
+      compound_ids = model.training_dataset.compound_ids
+      predictions = model.predict model.training_dataset.compounds
+      predictions = predictions.each_with_index {|p,i| p[:compound_id] = compound_ids[i]}
+      predictions.select!{|p| p[:database_activities] and !p[:database_activities].empty?}
+      loo.nr_instances = predictions.size
+      predictions.select!{|p| p[:value]} # remove unpredicted
+      loo.predictions = predictions#.sort{|a,b| b[:confidence] <=> a[:confidence]}
+      loo.nr_unpredicted = loo.nr_instances - loo.predictions.size
+      loo.statistics
+      loo.save
+      loo
+    end
+
+    def model
+      Model::Lazar.find model_id
+    end
+  end
+
+  class ClassificationLeaveOneOutValidation < LeaveOneOutValidation
+
+    field :accept_values, type: Array
+    field :confusion_matrix, type: Array, default: []
+    field :weighted_confusion_matrix, type: Array, default: []
+    field :accuracy, type: Float
+    field :weighted_accuracy, type: Float
+    field :true_rate, type: Hash, default: {}
+    field :predictivity, type: Hash, default: {}
+    field :confidence_plot_id, type: BSON::ObjectId
+
+    def statistics
+      accept_values = Feature.find(model.prediction_feature_id).accept_values
+      confusion_matrix = Array.new(accept_values.size,0){Array.new(accept_values.size,0)}
+      weighted_confusion_matrix = Array.new(accept_values.size,0){Array.new(accept_values.size,0)}
+      predictions.each do |pred|
+        pred[:database_activities].each do |db_act|
+          if pred[:value]
+            if pred[:value] == db_act
+              if pred[:value] == accept_values[0]
+                confusion_matrix[0][0] += 1
+                weighted_confusion_matrix[0][0] += pred[:confidence]
+              elsif pred[:value] == accept_values[1]
+                confusion_matrix[1][1] += 1
+                weighted_confusion_matrix[1][1] += pred[:confidence]
+              end
+            else
+              if pred[:value] == accept_values[0]
+                confusion_matrix[0][1] += 1
+                weighted_confusion_matrix[0][1] += pred[:confidence]
+              elsif pred[:value] == accept_values[1]
+                confusion_matrix[1][0] += 1
+                weighted_confusion_matrix[1][0] += pred[:confidence]
+              end
+            end
+          end
+        end
+      end
+      accept_values.each_with_index do |v,i|
+        true_rate[v] = confusion_matrix[i][i]/confusion_matrix[i].reduce(:+).to_f
+        predictivity[v] = confusion_matrix[i][i]/confusion_matrix.collect{|n| n[i]}.reduce(:+).to_f
+      end
+      confidence_sum = 0
+      weighted_confusion_matrix.each do |r|
+        r.each do |c|
+          confidence_sum += c
+        end
+      end
+      update_attributes(
+        accept_values: accept_values,
+        confusion_matrix: confusion_matrix,
+        weighted_confusion_matrix: weighted_confusion_matrix,
+        accuracy: (confusion_matrix[0][0]+confusion_matrix[1][1])/(nr_instances-nr_unpredicted).to_f,
+        weighted_accuracy: (weighted_confusion_matrix[0][0]+weighted_confusion_matrix[1][1])/confidence_sum.to_f,
+        true_rate: true_rate,
+        predictivity: predictivity,
+        finished_at: Time.now
+      )
+      $logger.debug "Accuracy #{accuracy}"
+    end
+
+    def confidence_plot
+      unless confidence_plot_id
+        tmpfile = "/tmp/#{id.to_s}_confidence.svg"
+        accuracies = []
+        confidences = []
+        correct_predictions = 0
+        incorrect_predictions = 0
+        predictions.each do |p|
+          p[:database_activities].each do |db_act|
+            if p[:value] 
+              p[:value] == db_act ? correct_predictions += 1 : incorrect_predictions += 1
+              accuracies << correct_predictions/(correct_predictions+incorrect_predictions).to_f
+              confidences << p[:confidence]
+
+            end
+          end
+        end
+        R.assign "accuracy", accuracies
+        R.assign "confidence", confidences
+        R.eval "image = qplot(confidence,accuracy)+ylab('accumulated accuracy')+scale_x_reverse()"
+        R.eval "ggsave(file='#{tmpfile}', plot=image)"
+        file = Mongo::Grid::File.new(File.read(tmpfile), :filename => "#{self.id.to_s}_confidence_plot.svg")
+        plot_id = $gridfs.insert_one(file)
+        update(:confidence_plot_id => plot_id)
+      end
+      $gridfs.find_one(_id: confidence_plot_id).data
+    end
+  end
+  
+
+  class RegressionLeaveOneOutValidation < LeaveOneOutValidation
+
+
+    field :rmse, type: Float, default: 0.0
+    field :mae, type: Float, default: 0
+    #field :weighted_rmse, type: Float, default: 0
+    #field :weighted_mae, type: Float, default: 0
+    field :r_squared, type: Float
+    field :correlation_plot_id, type: BSON::ObjectId
+    field :confidence_plot_id, type: BSON::ObjectId
+
+    def statistics
+      confidence_sum = 0
+      predicted_values = []
+      measured_values = []
+      predictions.each do |pred|
+        pred[:database_activities].each do |activity|
+          if pred[:value]
+            predicted_values << pred[:value]
+            measured_values << activity
+            error = Math.log10(pred[:value])-Math.log10(activity)
+            self.rmse += error**2
+            #self.weighted_rmse += pred[:confidence]*error**2
+            self.mae += error.abs
+            #self.weighted_mae += pred[:confidence]*error.abs
+            #confidence_sum += pred[:confidence]
+          end
+        end
+        if pred[:database_activities].empty?
+          warnings << "No training activities for #{Compound.find(compound_id).smiles} in training dataset #{model.training_dataset_id}."
+          $logger.debug "No training activities for #{Compound.find(compound_id).smiles} in training dataset #{model.training_dataset_id}."
+        end
+      end
+      R.assign "measurement", measured_values
+      R.assign "prediction", predicted_values
+      R.eval "r <- cor(-log(measurement),-log(prediction),use='complete')"
+      r = R.eval("r").to_ruby
+
+      self.mae = self.mae/predictions.size
+      #self.weighted_mae = self.weighted_mae/confidence_sum
+      self.rmse = Math.sqrt(self.rmse/predictions.size)
+      #self.weighted_rmse = Math.sqrt(self.weighted_rmse/confidence_sum)
+      self.r_squared = r**2
+      self.finished_at = Time.now
+      save
+      $logger.debug "R^2 #{r**2}"
+      $logger.debug "RMSE #{rmse}"
+      $logger.debug "MAE #{mae}"
+    end
+
+    def correlation_plot
+      unless correlation_plot_id
+        tmpfile = "/tmp/#{id.to_s}_correlation.svg"
+        predicted_values = []
+        measured_values = []
+        predictions.each do |pred|
+          pred[:database_activities].each do |activity|
+            if pred[:value]
+              predicted_values << pred[:value]
+              measured_values << activity
+            end
+          end
+        end
+        attributes = Model::Lazar.find(self.model_id).attributes
+        attributes.delete_if{|key,_| key.match(/_id|_at/) or ["_id","creator","name"].include? key}
+        attributes = attributes.values.collect{|v| v.is_a?(String) ? v.sub(/OpenTox::/,'') : v}.join("\n")
+        R.assign "measurement", measured_values
+        R.assign "prediction", predicted_values
+        R.eval "all = c(-log(measurement),-log(prediction))"
+        R.eval "range = c(min(all), max(all))"
+        R.eval "image = qplot(-log(prediction),-log(measurement),main='#{self.name}',asp=1,xlim=range, ylim=range)"
+        R.eval "image = image + geom_abline(intercept=0, slope=1)"
+        R.eval "ggsave(file='#{tmpfile}', plot=image)"
+        file = Mongo::Grid::File.new(File.read(tmpfile), :filename => "#{self.id.to_s}_correlation_plot.svg")
+        plot_id = $gridfs.insert_one(file)
+        update(:correlation_plot_id => plot_id)
+      end
+      $gridfs.find_one(_id: correlation_plot_id).data
+    end
+  end
+
+end

data/lib/model.rb

@@ -2,51 +2,79 @@ module OpenTox
 
   module Model
 
-    class Lazar 
+    class Model
       include OpenTox
       include Mongoid::Document
       include Mongoid::Timestamps
       store_in collection: "models"
 
-      field :title, as: :name, type: String
+      field :name, type: String
       field :creator, type: String, default: __FILE__
       # datasets
       field :training_dataset_id, type: BSON::ObjectId
       # algorithms
       field :prediction_algorithm, type: String
-      field :neighbor_algorithm, type: String
-      field :neighbor_algorithm_parameters, type: Hash
       # prediction feature
       field :prediction_feature_id, type: BSON::ObjectId
 
-      attr_accessor :prediction_dataset
-      attr_accessor :training_dataset
+      def training_dataset
+        Dataset.find(training_dataset_id)
+      end
+    end
+
+    class Lazar < Model
+
+      # algorithms
+      field :neighbor_algorithm, type: String
+      field :neighbor_algorithm_parameters, type: Hash, default: {}
 
       # Create a lazar model from a training_dataset and a feature_dataset
       # @param [OpenTox::Dataset] training_dataset
       # @return [OpenTox::Model::Lazar] Regression or classification model
-      def self.create training_dataset
+      def initialize training_dataset, params={}
 
-        bad_request_error "More than one prediction feature found in training_dataset #{training_dataset.id}" unless training_dataset.features.size == 1
+        super params
 
         # TODO document convention
         prediction_feature = training_dataset.features.first
-        prediction_feature.nominal ?  lazar = OpenTox::Model::LazarClassification.new : lazar = OpenTox::Model::LazarRegression.new
-        lazar.training_dataset_id = training_dataset.id
-        lazar.prediction_feature_id = prediction_feature.id
-        lazar.title = prediction_feature.title 
+        # set defaults for empty parameters
+        self.prediction_feature_id ||= prediction_feature.id
+        self.training_dataset_id ||= training_dataset.id
+        self.name ||= "#{training_dataset.name} #{prediction_feature.name}" 
+        self.neighbor_algorithm_parameters ||= {}
+        self.neighbor_algorithm_parameters[:training_dataset_id] = training_dataset.id
+        save
+        self
+      end
 
-        lazar.save
-        lazar
+      def predict_compound compound
+        prediction_feature = Feature.find prediction_feature_id
+        neighbors = compound.send(neighbor_algorithm, neighbor_algorithm_parameters)
+        # remove neighbors without prediction_feature
+        # check for database activities (neighbors may include query compound)
+        database_activities = nil
+        prediction = {}
+        if neighbors.collect{|n| n["_id"]}.include? compound.id
+
+          database_activities = neighbors.select{|n| n["_id"] == compound.id}.first["features"][prediction_feature.id.to_s].uniq
+          prediction[:database_activities] = database_activities
+          prediction[:warning] = "#{database_activities.size} compounds have been removed from neighbors, because they have the same structure as the query compound."
+          neighbors.delete_if{|n| n["_id"] == compound.id}
+        end
+        neighbors.delete_if{|n| n['features'].empty? or n['features'][prediction_feature.id.to_s] == [nil] }
+        if neighbors.empty?
+          prediction.merge!({:value => nil,:confidence => nil,:warning => "Could not find similar compounds with experimental data in the training dataset.",:neighbors => []})
+        else
+          prediction.merge!(Algorithm.run(prediction_algorithm, compound, {:neighbors => neighbors,:training_dataset_id=> training_dataset_id,:prediction_feature_id => prediction_feature.id}))
+          prediction[:neighbors] = neighbors
+          prediction[:neighbors] ||= []
+        end
+        prediction
       end
 
       def predict object
 
-        t = Time.now
-        at = Time.now
-
         training_dataset = Dataset.find training_dataset_id
-        prediction_feature = Feature.find prediction_feature_id
 
         # parse data
         compounds = []
@@ -63,50 +91,33 @@ module OpenTox
 
         # make predictions
         predictions = []
-        neighbors = []
-        compounds.each_with_index do |compound,c|
-          t = Time.new
-          database_activities = training_dataset.values(compound,prediction_feature)
-          if database_activities and !database_activities.empty?
-            database_activities = database_activities.first if database_activities.size == 1
-            predictions << {:compound => compound, :value => database_activities, :confidence => "measured", :warning => "Compound #{compound.smiles} occurs in training dataset with activity '#{database_activities}'."}
-            next
-          end
-          neighbors = Algorithm.run(neighbor_algorithm, compound, neighbor_algorithm_parameters)
-          # add activities
-          # TODO: improve efficiency, takes 3 times longer than previous version
-          neighbors.collect! do |n|
-            rows = training_dataset.compound_ids.each_index.select{|i| training_dataset.compound_ids[i] == n.first}
-            acts = rows.collect{|row| training_dataset.data_entries[row][0]}.compact
-            acts.empty? ? nil : n << acts
-          end
-          neighbors.compact! # remove neighbors without training activities
-          predictions << Algorithm.run(prediction_algorithm, neighbors)
-        end 
+        predictions = compounds.collect{|c| predict_compound c}
 
         # serialize result
         case object.class.to_s
         when "OpenTox::Compound"
           prediction = predictions.first
-          prediction[:neighbors] = neighbors.sort{|a,b| b[1] <=> a[1]} # sort according to similarity
+          prediction[:neighbors].sort!{|a,b| b[1] <=> a[1]} # sort according to similarity
           return prediction
         when "Array"
           return predictions
         when "OpenTox::Dataset"
           # prepare prediction dataset
+          measurement_feature = Feature.find prediction_feature_id
+
+          prediction_feature = OpenTox::NumericFeature.find_or_create_by( "name" => measurement_feature.name + " (Prediction)" )
           prediction_dataset = LazarPrediction.new(
-            :title => "Lazar prediction for #{prediction_feature.title}",
+            :name => "Lazar prediction for #{prediction_feature.name}",
             :creator =>  __FILE__,
             :prediction_feature_id => prediction_feature.id
 
           )
-          confidence_feature = OpenTox::NumericFeature.find_or_create_by( "title" => "Prediction confidence" )
-          # TODO move into warnings field
-          warning_feature = OpenTox::NominalFeature.find_or_create_by("title" => "Warnings")
-          prediction_dataset.features = [ prediction_feature, confidence_feature, warning_feature ]
+          confidence_feature = OpenTox::NumericFeature.find_or_create_by( "name" => "Model RMSE" )
+          warning_feature = OpenTox::NominalFeature.find_or_create_by("name" => "Warnings")
+          prediction_dataset.features = [ prediction_feature, confidence_feature, measurement_feature, warning_feature ]
           prediction_dataset.compounds = compounds
-          prediction_dataset.data_entries = predictions.collect{|p| [p[:value], p[:confidence], p[:warning]]}
-          prediction_dataset.save_all
+          prediction_dataset.data_entries = predictions.collect{|p| [p[:value], p[:rmse] , p[:dataset_activities].to_s, p[:warning]]}
+          prediction_dataset.save
           return prediction_dataset
         end
 
@@ -120,26 +131,19 @@ module OpenTox
     end
 
     class LazarClassification < Lazar
-      def initialize
-        super
-        self.prediction_algorithm = "OpenTox::Algorithm::Classification.weighted_majority_vote"
-        self.neighbor_algorithm = "OpenTox::Algorithm::Neighbor.fingerprint_similarity"
-        self.neighbor_algorithm_parameters = {:min_sim => 0.7}
-      end
-    end
-
-    class LazarFminerClassification < LazarClassification
-
-      def self.create training_dataset
-        model = super(training_dataset)
-        model.update "_type" => self.to_s # adjust class
-        model = self.find model.id # adjust class
-        model.neighbor_algorithm = "OpenTox::Algorithm::Neighbor.fminer_similarity"
-        model.neighbor_algorithm_parameters = {
-          :feature_calculation_algorithm => "OpenTox::Algorithm::Descriptor.smarts_match",
-          :feature_dataset_id => Algorithm::Fminer.bbrc(training_dataset).id,
-          :min_sim => 0.3
-        }
+      
+      def self.create training_dataset, params={}
+        model = self.new training_dataset, params
+        model.prediction_algorithm = "OpenTox::Algorithm::Classification.weighted_majority_vote" unless model.prediction_algorithm
+        model.neighbor_algorithm ||= "fingerprint_neighbors"
+        model.neighbor_algorithm_parameters ||= {}
+        {
+          :type => "MP2D",
+          :training_dataset_id => training_dataset.id,
+          :min_sim => 0.1
+        }.each do |key,value|
+          model.neighbor_algorithm_parameters[key] ||= value
+        end
         model.save
         model
       end
@@ -147,20 +151,27 @@ module OpenTox
 
     class LazarRegression < Lazar
 
-      def initialize
-        super
-        self.neighbor_algorithm = "OpenTox::Algorithm::Neighbor.fingerprint_similarity"
-        self.prediction_algorithm = "OpenTox::Algorithm::Regression.weighted_average" 
-        self.neighbor_algorithm_parameters = {:min_sim => 0.7}
+      def self.create training_dataset, params={}
+        model = self.new training_dataset, params
+        model.neighbor_algorithm ||= "fingerprint_neighbors"
+        model.prediction_algorithm ||= "OpenTox::Algorithm::Regression.local_fingerprint_regression" 
+        model.neighbor_algorithm_parameters ||= {}
+        {
+          :type => "MP2D",
+          :training_dataset_id => training_dataset.id,
+          :min_sim => 0.1
+        }.each do |key,value|
+          model.neighbor_algorithm_parameters[key] ||= value
+        end
+        model.save
+        model
       end
-
     end
 
-    class PredictionModel
+    class Prediction
       include OpenTox
       include Mongoid::Document
       include Mongoid::Timestamps
-      store_in collection: "models"
 
       # TODO field Validations
       field :endpoint, type: String
@@ -168,10 +179,54 @@ module OpenTox
       field :source, type: String
       field :unit, type: String
       field :model_id, type: BSON::ObjectId
-      field :crossvalidation_id, type: BSON::ObjectId
+      field :repeated_crossvalidation_id, type: BSON::ObjectId
+
+      def predict object
+        Lazar.find(model_id).predict object
+      end
+
+      def training_dataset
+        model.training_dataset
+      end
+
+      def model
+        Lazar.find model_id
+      end
+
+      def repeated_crossvalidation
+        RepeatedCrossValidation.find repeated_crossvalidation_id
+      end
+
+      def crossvalidations
+        repeated_crossvalidation.crossvalidations
+      end
+
+      def regression?
+        training_dataset.features.first.numeric?
+      end
+
+      def classification?
+        training_dataset.features.first.nominal?
+      end
+
+      def self.from_csv_file file
+        metadata_file = file.sub(/csv$/,"json")
+        bad_request_error "No metadata file #{metadata_file}" unless File.exist? metadata_file
+        prediction_model = self.new JSON.parse(File.read(metadata_file))
+        training_dataset = Dataset.from_csv_file file
+        model = nil
+        if training_dataset.features.first.nominal?
+          model = LazarClassification.create training_dataset
+        elsif training_dataset.features.first.numeric?
+          model = LazarRegression.create training_dataset
+        end
+        prediction_model[:model_id] = model.id
+        prediction_model[:repeated_crossvalidation_id] = RepeatedCrossValidation.create(model).id
+        prediction_model.save
+        prediction_model
+      end
     end
 
   end
 
 end
-

data/lib/opentox.rb

@@ -12,8 +12,8 @@ module OpenTox
       include Mongoid::Document
       include Mongoid::Timestamps
       store_in collection: klass.downcase.pluralize
-      field :title, as: :name,  type: String
-
+      field :name,  type: String
+      field :warnings, type: Array, default: []
     end
     OpenTox.const_set klass,c
   end

data/lib/overwrite.rb

@@ -9,6 +9,11 @@ class Object
   def numeric?
     true if Float(self) rescue false
   end
+
+  # Returns dimension of nested arrays
+  def dimension
+    self.class == Array ? 1 + self[0].dimension : 0
+  end
 end
 
 class Numeric
@@ -17,6 +22,14 @@ class Numeric
   end
 end
 
+class Float
+  # round to n significant digits
+  # http://stackoverflow.com/questions/8382619/how-to-round-a-float-to-a-specified-number-of-significant-digits-in-ruby
+  def signif(n)
+    Float("%.#{n}g" % self)
+  end
+end
+
 module Enumerable
   # @return [Array] only the duplicates of an enumerable
   def duplicates
@@ -81,6 +94,26 @@ class Array
     return self.uniq.size == 1
   end
 
+  def median
+    sorted = self.sort
+    len = sorted.length
+    (sorted[(len - 1) / 2] + sorted[len / 2]) / 2.0
+  end
+
+  def mean
+    self.inject{ |sum, el| sum + el }.to_f / self.size
+  end
+
+  def sample_variance
+    m = self.mean
+    sum = self.inject(0){|accum, i| accum +(i-m)**2 }
+    sum/(self.length - 1).to_f
+  end
+
+  def standard_deviation
+    Math.sqrt(self.sample_variance)
+  end
+
 end
 
 module URI
@@ -116,4 +149,8 @@ module URI
     false
   end
 
+  def self.task? uri
+    uri =~ /task/ and URI.valid? uri
+  end
+
 end