lazar 1.0.0 → 1.0.1

data/lib/validation-statistics.rb

@@ -1,7 +1,10 @@
 module OpenTox
   module Validation
+    # Statistical evaluation of classification validations
     module ClassificationStatistics
 
+      # Get statistics
+      # @return [Hash]
       def statistics 
         self.accept_values = model.prediction_feature.accept_values
         self.confusion_matrix = Array.new(accept_values.size){Array.new(accept_values.size,0)}
@@ -63,6 +66,9 @@ module OpenTox
         }
       end
 
+      # Plot accuracy vs prediction probability
+      # @param [String,nil] format
+      # @return [Blob]
       def probability_plot format: "pdf"
         #unless probability_plot_id
 
@@ -99,8 +105,11 @@ module OpenTox
       end
     end
 
+    # Statistical evaluation of regression validations
     module RegressionStatistics
 
+      # Get statistics
+      # @return [Hash]
       def statistics
         self.rmse = 0
         self.mae = 0
@@ -147,10 +156,15 @@ module OpenTox
         }
       end
 
+      # Get percentage of measurements within the prediction interval
+      # @return [Float]
       def percent_within_prediction_interval
         100*within_prediction_interval.to_f/(within_prediction_interval+out_of_prediction_interval)
       end
 
+      # Plot predicted vs measured values
+      # @param [String,nil] format
+      # @return [Blob]
       def correlation_plot format: "png"
         unless correlation_plot_id
           tmpfile = "/tmp/#{id.to_s}_correlation.#{format}"
@@ -177,6 +191,11 @@ module OpenTox
         $gridfs.find_one(_id: correlation_plot_id).data
       end
 
+      # Get predictions with the largest difference between predicted and measured values
+      # @params [Fixnum] number of predictions
+      # @params [TrueClass,FalseClass,nil] include neighbors
+      # @params [TrueClass,FalseClass,nil] show common descriptors
+      # @return [Hash]
       def worst_predictions n: 5, show_neigbors: true, show_common_descriptors: false
         worst_predictions = predictions.sort_by{|sid,p| -(p["value"] - p["measurements"].median).abs}[0,n]
         worst_predictions.collect do |p|

data/lib/validation.rb

@@ -2,6 +2,7 @@ module OpenTox
 
   module Validation
 
+    # Base validation class
     class Validation
       include OpenTox
       include Mongoid::Document
@@ -14,6 +15,8 @@ module OpenTox
       field :predictions, type: Hash, default: {}
       field :finished_at, type: Time 
 
+      # Get model
+      # @return [OpenTox::Model::Lazar]
       def model
         Model::Lazar.find model_id
       end

data/test/feature.rb

@@ -55,7 +55,7 @@ class FeatureTest < MiniTest::Test
   end
 
   def test_physchem_description
-    assert_equal 355, PhysChem.descriptors.size
+    assert_equal 346, PhysChem.descriptors.size
     assert_equal 15, PhysChem.openbabel_descriptors.size
     assert_equal 295, PhysChem.cdk_descriptors.size
     assert_equal 45, PhysChem.joelib_descriptors.size
@@ -63,7 +63,7 @@ class FeatureTest < MiniTest::Test
   end
 
   def test_physchem
-    assert_equal 355, PhysChem.descriptors.size
+    assert_equal 346, PhysChem.descriptors.size
     c = Compound.from_smiles "CC(=O)CC(C)C"
     logP = PhysChem.find_or_create_by :name => "Openbabel.logP"
     assert_equal 1.6215, logP.calculate(c)

data/test/model-nanoparticle.rb

@@ -8,6 +8,13 @@ class NanoparticleModelTest  < MiniTest::Test
     @prediction_feature = @training_dataset.features.select{|f| f["name"] == 'log2(Net cell association)'}.first
   end
 
+  def test_core_coating_source_uris
+    @training_dataset.nanoparticles.each do |np|
+      refute_nil np.core.source
+      np.coating.each{|c| refute_nil c.source}
+    end
+  end
+
   def test_nanoparticle_model
     assert true, @prediction_feature.measured
     model = Model::Lazar.create training_dataset: @training_dataset, prediction_feature: @prediction_feature

data/test/nanomaterial-model-validation.rb

@@ -8,7 +8,7 @@ class NanomaterialValidationModelTest < MiniTest::Test
   end
 
   def test_default_nanomaterial_validation_model
-    validation_model = Model::NanoValidation.create
+    validation_model = Model::Validation.from_enanomapper
     [:endpoint,:species,:source].each do |p|
       refute_empty validation_model[p]
     end
@@ -39,7 +39,7 @@ class NanomaterialValidationModelTest < MiniTest::Test
       :prediction => { :method => "OpenTox::Algorithm::Regression.weighted_average" },
       :feature_selection => nil
     }
-    validation_model = Model::NanoValidation.create algorithms: algorithms
+    validation_model = Model::Validation.from_enanomapper algorithms: algorithms
     assert validation_model.regression?
     refute validation_model.classification?
     validation_model.crossvalidations.each do |cv|
@@ -50,6 +50,5 @@ class NanomaterialValidationModelTest < MiniTest::Test
     assert_includes nanoparticle.dataset_ids, @training_dataset.id
     prediction = validation_model.predict nanoparticle
     refute_nil prediction[:value]
-    assert_includes prediction[:prediction_interval][0]..prediction[:prediction_interval][1], prediction[:measurements].median, "This assertion assures that measured values are within the prediction interval. It may fail in 5% of the predictions."
   end
 end

data/test/setup.rb

@@ -6,8 +6,4 @@ include OpenTox
 TEST_DIR ||= File.expand_path(File.dirname(__FILE__))
 DATA_DIR ||= File.join(TEST_DIR,"data")
 training_dataset = Dataset.where(:name => "Protein Corona Fingerprinting Predicts the Cellular Interaction of Gold and Silver Nanoparticles").first
-unless training_dataset
-  Import::Enanomapper.import File.join(File.dirname(__FILE__),"data","enm")
-end
-#$mongo.database.drop
-#$gridfs = $mongo.database.fs
+Import::Enanomapper.import unless training_dataset

data/test/validation-regression.rb

@@ -83,10 +83,9 @@ class ValidationRegressionTest < MiniTest::Test
     model = Model::Lazar.create training_dataset: dataset
     repeated_cv = RepeatedCrossValidation.create model
     repeated_cv.crossvalidations.each do |cv|
-      #assert cv.r_squared > 0.34, "R^2 (#{cv.r_squared}) should be larger than 0.034"
-      #assert_operator cv.accuracy, :>, 0.7, "model accuracy < 0.7, this may happen by chance due to an unfavorable training/test set split"
+      assert cv.r_squared > 0.34, "R^2 (#{cv.r_squared}) should be larger than 0.034"
+      assert_operator cv.accuracy, :>, 0.7, "model accuracy < 0.7, this may happen by chance due to an unfavorable training/test set split"
     end
-    File.open("tmp.png","w+"){|f| f.puts repeated_cv.correlation_plot}
   end
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lazar
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Christoph Helma, Martin Guetlein, Andreas Maunz, Micha Rautenberg, David Vorgrimmler,
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-12-21 00:00:00.000000000 Z
+date: 2017-01-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -134,7 +134,6 @@ files:
 - lib/crossvalidation.rb
 - lib/dataset.rb
 - lib/error.rb
-- lib/experiment.rb
 - lib/feature.rb
 - lib/feature_selection.rb
 - lib/import.rb
@@ -222,8 +221,54 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: lazar
-rubygems_version: 2.5.1
+rubygems_version: 2.5.2
 signing_key: 
 specification_version: 4
 summary: Lazar framework
-test_files: []
+test_files:
+- test/all.rb
+- test/compound.rb
+- test/data/EPAFHM.csv
+- test/data/EPAFHM.medi.csv
+- test/data/EPAFHM.medi_log10.csv
+- test/data/EPAFHM.mini.csv
+- test/data/EPAFHM.mini_log10.csv
+- test/data/EPAFHM_log10.csv
+- test/data/ISSCAN-multi.csv
+- test/data/LOAEL_mmol_corrected_smiles.csv
+- test/data/acetaldehyde.sdf
+- test/data/batch_prediction.csv
+- test/data/batch_prediction_inchi_small.csv
+- test/data/batch_prediction_smiles_small.csv
+- test/data/hamster_carcinogenicity.csv
+- test/data/hamster_carcinogenicity.json
+- test/data/hamster_carcinogenicity.mini.bool_float.csv
+- test/data/hamster_carcinogenicity.mini.bool_int.csv
+- test/data/hamster_carcinogenicity.mini.bool_string.csv
+- test/data/hamster_carcinogenicity.mini.csv
+- test/data/hamster_carcinogenicity_with_errors.csv
+- test/data/kazius.csv
+- test/data/loael.csv
+- test/data/loael_log10.csv
+- test/data/multi_cell_call.csv
+- test/data/multi_cell_call_no_dup.csv
+- test/data/multicolumn.csv
+- test/data/rat_feature_dataset.csv
+- test/data/wrong_dataset.csv
+- test/dataset.rb
+- test/default_environment.rb
+- test/descriptor.rb
+- test/error.rb
+- test/experiment.rb
+- test/feature.rb
+- test/gridfs.rb
+- test/model-classification.rb
+- test/model-nanoparticle.rb
+- test/model-regression.rb
+- test/model-validation.rb
+- test/nanomaterial-model-validation.rb
+- test/setup.rb
+- test/test_environment.rb
+- test/validation-classification.rb
+- test/validation-nanoparticle.rb
+- test/validation-regression.rb

data/lib/experiment.rb

@@ -1,99 +0,0 @@
-module OpenTox
-
-  class Experiment
-    field :dataset_ids, type: Array
-    field :model_settings, type: Array, default: []
-    field :results, type: Hash, default: {}
-
-    def run 
-      dataset_ids.each do |dataset_id|
-        dataset = Dataset.find(dataset_id)
-        results[dataset_id.to_s] = []
-        model_settings.each do |setting|
-          setting = setting.dup
-          model_algorithm = setting.delete :model_algorithm #if setting[:model_algorithm]
-          model = Object.const_get(model_algorithm).create dataset, setting
-          $logger.debug model
-          model.save
-          repeated_crossvalidation = RepeatedCrossValidation.create model
-          results[dataset_id.to_s] << {:model_id => model.id, :repeated_crossvalidation_id => repeated_crossvalidation.id}
-        end
-      end
-      save
-    end
-
-    def report
-      # statistical significances http://www.r-bloggers.com/anova-and-tukeys-test-on-r/
-      report = {}
-      report[:name] = name
-      report[:experiment_id] = self.id.to_s
-      report[:results] = {}
-      parameters = []
-      dataset_ids.each do |dataset_id|
-        dataset_name = Dataset.find(dataset_id).name
-        report[:results][dataset_name] = {}
-        report[:results][dataset_name][:anova] = {}
-        report[:results][dataset_name][:data] = []
-        # TODO results[dataset_id.to_s] does not exist
-        results[dataset_id.to_s].each do |result|
-          model = Model::Lazar.find(result[:model_id])
-          repeated_cv = RepeatedCrossValidation.find(result[:repeated_crossvalidation_id])
-          crossvalidations = repeated_cv.crossvalidations
-          if crossvalidations.first.is_a? ClassificationCrossValidation
-            parameters = [:accuracy,:true_rate,:predictivity]
-          elsif crossvalidations.first.is_a? RegressionCrossValidation
-            parameters = [:rmse,:mae,:r_squared]
-          end
-          summary = {}
-          [:neighbor_algorithm, :neighbor_algorithm_parameters, :prediction_algorithm].each do |key|
-            summary[key] = model[key]
-          end
-          summary[:nr_instances] = crossvalidations.first.nr_instances
-          summary[:nr_unpredicted] = crossvalidations.collect{|cv| cv.nr_unpredicted}
-          summary[:time] = crossvalidations.collect{|cv| cv.time}
-          parameters.each do |param|
-            summary[param] = crossvalidations.collect{|cv| cv.send(param)}
-          end
-          report[:results][dataset_name][:data] << summary
-        end
-      end
-      report[:results].each do |dataset,results|
-        ([:time,:nr_unpredicted]+parameters).each do |param|
-          experiments = []
-          outcome = []
-          results[:data].each_with_index do |result,i|
-            result[param].each do |p|
-              experiments << i
-              p = nil if p.kind_of? Float and p.infinite? # TODO fix @ division by 0
-              outcome << p
-            end
-          end
-          begin
-          R.assign "experiment_nr",experiments.collect{|i| "Experiment #{i}"}
-          R.eval "experiment_nr = factor(experiment_nr)"
-          R.assign "outcome", outcome
-          R.eval "data = data.frame(experiment_nr,outcome)"
-          # one-way ANOVA
-          R.eval "fit = aov(outcome ~ experiment_nr, data=data,na.action='na.omit')"
-          # http://stackoverflow.com/questions/3366506/extract-p-value-from-aov
-          p_value = R.eval("summary(fit)[[1]][['Pr(>F)']][[1]]").to_ruby
-          # aequivalent
-          # sum = R.eval("summary(fit)")
-          #p_value = sum.to_ruby.first.last.first
-          rescue 
-            p_value = nil
-          end
-          report[:results][dataset][:anova][param] = p_value
-=begin
-=end
-        end
-      end
-      report
-    end
-
-    def summary
-      report[:results].collect{|dataset,data| {dataset => data[:anova].select{|param,p_val| p_val < 0.1}}}
-    end
-  end
-
-end