lazar 0.9.3 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 44e8fb9b8d65ca3f2fb8d02fb08c76e98ebc140c
-  data.tar.gz: d62b490434324e405ad10a13b72fc51574e02404
+  metadata.gz: 2211d5cf1767b241583acff9a22379b56a5d8f1c
+  data.tar.gz: 923a3d00d5c78fd77a2153c973c5e3935c939eda
 SHA512:
-  metadata.gz: 2ea37844e810a1410453e36b87e9d4473226bd78a57f692f8f46f8c56153fec13cb3a320c3f0df718242bca7aed13bebb510192812ee10ce41c3acd1a36d8c92
-  data.tar.gz: 696378dea89f26a8a50c96e20de20a4fcbc8a717a22c3198ba352f06b92c4d597c8a5903352f249ab0e9af295aee803f60f37a0755253cee9bc7a7f5ce5556dd
+  metadata.gz: 2a366bae505c427a72211df4d59c7f296ead656bfe3f42db0fb6bb2dc3885028c70ba9df0aa7778c0bd78acdbd7b2939417caafd342a535c4954a34fef410c8d
+  data.tar.gz: 04fd93e7ab52517d338e6005223fe22b498d74be324f8dc6ef2e3a4d4a843202abc9224ff55e8ba053ce7a16a6a76301437f4fc061ac2719d65ff3afa392396a

data/.gitignore

@@ -1,8 +1,5 @@
-last-utils
-libfminer
+R
 openbabel
-fminer_debug.txt
-test/fminer_debug.txt
 Gemfile.lock
 *.gem
 .bundle

data/README.md

@@ -6,31 +6,21 @@ Ruby libraries for the lazar framework
 Dependencies
 ------------
 
-  lazar depends on a couple of external programs and libraries. On Debian 7 "Wheezy" systems you can install them with
-
-   `sudo apt-get install build-essential ruby ruby-dev git cmake swig r-base r-base-dev r-cran-rserve openjdk-7-jre libgsl0-dev libxml2-dev zlib1g-dev libcairo2-dev`
-  
-  You will also need at least mongodb version 3.0, but Debian "Wheezy" provides version 2.4. Please follow the instructions at http://docs.mongodb.org/manual/tutorial/install-mongodb-on-debian/:
-
-  ```
-  sudo apt-key adv --keyserver keyserver.ubuntu.com --recv 7F0CEB10
-  echo "deb http://repo.mongodb.org/apt/debian wheezy/mongodb-org/3.0 main" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.0.list
-  sudo apt-get update
-  sudo apt-get install -y mongodb-org
-  ```
+  lazar depends on a couple of external programs and libraries. All required libraries will be installed with the `gem install lazar` command. 
+  If any of the dependencies fails to install, please check if all required development packages are installed from your operating systems package manager (e.g. `apt`, `rpm`, `pacman`, ...). 
+  You will need a working Java runtime to use descriptor calculation algorithms from CDK and JOELib libraries.
 
 Installation
 ------------
 
   `gem install lazar`
 
-  Please be patient, the compilation of OpenBabel and Fminer libraries can be very time consuming. If installation fails you can try to install manually:
+  Please be patient, the compilation of external libraries can be very time consuming. If installation fails you can try to install manually:
 
   ```
   git clone https://github.com/opentox/lazar.git
   cd lazar
   ruby ext/lazar/extconf.rb
-  sudo Rscript ext/lazar/rinstall.R
   bundle install
   ```
 
@@ -42,4 +32,4 @@ Documentation
 
 Copyright
 ---------
-Copyright (c) 2009-2015 Christoph Helma, Martin Guetlein, Micha Rautenberg, Andreas Maunz, David Vorgrimmler, Denis Gebele. See LICENSE for details.
+Copyright (c) 2009-2016 Christoph Helma, Martin Guetlein, Micha Rautenberg, Andreas Maunz, David Vorgrimmler, Denis Gebele. See LICENSE for details.

data/VERSION

@@ -1 +1 @@
-0.9.3
+1.0.0

data/ext/lazar/extconf.rb

@@ -15,7 +15,7 @@ abort "Please install Rserve on your system. Execute 'install.packages('Rserve')
 # install R packages
 r_dir = File.join main_dir, "R"
 FileUtils.mkdir_p r_dir
-FileUtils.mkdir_p File.join(main_dir,"bin") # for Rserve binary
+#FileUtils.mkdir_p File.join(main_dir,"bin") # for Rserve binary
 rinstall = File.expand_path(File.join(File.dirname(__FILE__),"rinstall.R"))
 puts `Rscript --vanilla #{rinstall} #{r_dir}`
 

data/ext/lazar/rinstall.R

@@ -1,10 +1,12 @@
 libdir = commandArgs(trailingOnly=TRUE)[1]
 repo = "https://stat.ethz.ch/CRAN/"
 #install.packages("Rserve",lib=libdir,repos=repo,dependencies=TRUE)
-install.packages("iterators",lib=libdir,repos=repo);
-install.packages("foreach",lib=libdir,repos=repo);
-install.packages("gridExtra",lib=libdir,repos=repo);
-install.packages("ggplot2",lib=libdir,repos=repo);
-install.packages("pls",lib=libdir,repos=repo);
-install.packages("caret",lib=libdir,repos=repo);
-install.packages("doMC",lib=libdir,repos=repo);
+install.packages("stringi",lib=libdir,repos=repo,dependencies=TRUE);
+install.packages("iterators",lib=libdir,repos=repo,dependencies=TRUE);
+install.packages("foreach",lib=libdir,repos=repo,dependencies=TRUE);
+install.packages("gridExtra",lib=libdir,repos=repo,dependencies=TRUE);
+install.packages("ggplot2",lib=libdir,repos=repo,dependencies=TRUE);
+install.packages("pls",lib=libdir,repos=repo,dependencies=TRUE);
+install.packages("randomForest",lib=libdir,repos=repo,dependencies=TRUE);
+install.packages("caret",lib=libdir,repos=repo,dependencies=TRUE);
+install.packages("doMC",lib=libdir,repos=repo,dependencies=TRUE);

data/java/CdkDescriptorInfo.java

@@ -1,11 +1,12 @@
 import java.util.*;
-import org.openscience.cdk.qsar.descriptors.molecular.*;
+import org.openscience.cdk.DefaultChemObjectBuilder;
 import org.openscience.cdk.qsar.*;
+//import org.openscience.cdk.qsar.descriptors.molecular.*;
 
 class CdkDescriptorInfo {
   public static void main(String[] args) {
 
-    DescriptorEngine engine = new DescriptorEngine(DescriptorEngine.MOLECULAR);
+    DescriptorEngine engine = new DescriptorEngine(IMolecularDescriptor.class,null);
 
     for (Iterator<IDescriptor> it = engine.getDescriptorInstances().iterator(); it.hasNext(); ) {
       IDescriptor descriptor = it.next();

data/java/CdkDescriptors.java

@@ -1,10 +1,10 @@
 import java.util.*;
 import java.io.*;
 import org.openscience.cdk.DefaultChemObjectBuilder;
-import org.openscience.cdk.interfaces.IMolecule;
-import org.openscience.cdk.io.iterator.IteratingMDLReader;
+import org.openscience.cdk.IImplementationSpecification;
+import org.openscience.cdk.interfaces.IAtomContainer;
+import org.openscience.cdk.io.iterator.IteratingSDFReader;
 import org.openscience.cdk.qsar.*;
-import org.openscience.cdk.qsar.DescriptorValue;
 import org.openscience.cdk.aromaticity.CDKHueckelAromaticityDetector;
 import org.openscience.cdk.tools.manipulator.AtomContainerManipulator;
 import org.openscience.cdk.exception.NoSuchAtomTypeException;
@@ -17,8 +17,8 @@ class CdkDescriptors {
 	System.exit(1);
     }
     if (! new File(args[0]).exists()){
-	System.err.println("file not found "+args[0]);
-	System.exit(1);
+      System.err.println("file not found "+args[0]);
+      System.exit(1);
     }
 
     // command line descriptor params can be either "descriptorName" or "descriptorValueName"
@@ -34,19 +34,19 @@ class CdkDescriptors {
     for (int i =1; i < args.length; i++) {
       String descriptorName;
       if (args[i].indexOf(".")!=-1) {
-          descriptorValueNames.add(args[i]);
-	  descriptorName = args[i].substring(0,args[i].indexOf("."));
+        descriptorValueNames.add(args[i]);
+        descriptorName = args[i].substring(0,args[i].indexOf("."));
       }
       else {
-	  descriptorNames.add(args[i]);
-          descriptorName = args[i];
+        descriptorNames.add(args[i]);
+        descriptorName = args[i];
       }
       classNames.add(getDescriptorClassName(descriptorName));
     }
 
-    engine = new DescriptorEngine(new ArrayList<String>(classNames));
+    engine = new DescriptorEngine(new ArrayList<String>(classNames),null);
     List<IDescriptor> instances =  engine.instantiateDescriptors(new ArrayList<String>(classNames));
-    List<DescriptorSpecification> specs = engine.initializeSpecifications(instances);
+    List<IImplementationSpecification> specs = engine.initializeSpecifications(instances);
     engine.setDescriptorInstances(instances);
     engine.setDescriptorSpecifications(specs);
 
@@ -54,13 +54,13 @@ class CdkDescriptors {
       BufferedReader br = new BufferedReader(new FileReader(args[0]));
       PrintWriter yaml = new PrintWriter(new FileWriter(args[0]+"cdk.yaml"));
       // parse 3d sdf from file and calculate descriptors
-      IteratingMDLReader reader = new IteratingMDLReader( br, DefaultChemObjectBuilder.getInstance());
+      IteratingSDFReader reader = new IteratingSDFReader( br, DefaultChemObjectBuilder.getInstance());
       int c = 0;
       while (reader.hasNext()) {
         try {
           System.out.println("computing "+(args.length-1)+" descriptors for compound "+(++c));
-          IMolecule molecule = (IMolecule)reader.next();
-          molecule = (IMolecule) AtomContainerManipulator.removeHydrogens(molecule);
+          IAtomContainer molecule = (IAtomContainer)reader.next();
+          molecule = (IAtomContainer) AtomContainerManipulator.removeHydrogens(molecule);
           try {
             AtomContainerManipulator.percieveAtomTypesAndConfigureAtoms(molecule);
           }
@@ -110,21 +110,21 @@ class CdkDescriptors {
      * problem: Descriptor is not always at the end of the class (APolDescriptor), but may be in the middle (AutocorrelationDescriptorPolarizability)
      * this method makes a class-lookup using trial and error */
     static String getDescriptorClassName(String descriptorName) {
-	String split = splitCamelCase(descriptorName)+" "; // space mark possible positions for 'Descriptor'
-	for(int i = split.length()-1; i>0; i--) {
-	    if (split.charAt(i)==' ') { // iterate over all spaces, starting with the trailing one
-		String test = split.substring(0,i)+"Descriptor"+split.substring(i+1,split.length()); // replace current space with 'Descriptor' ..
-		test = test.replaceAll("\\s",""); // .. and remove other spaces
-		String className = "org.openscience.cdk.qsar.descriptors.molecular." + test;
-		try {
-		    Class.forName(className);
-		    return className;
-		} catch (ClassNotFoundException e) {}
-	    }
+      String split = splitCamelCase(descriptorName)+" "; // space mark possible positions for 'Descriptor'
+      for(int i = split.length()-1; i>0; i--) {
+        if (split.charAt(i)==' ') { // iterate over all spaces, starting with the trailing one
+          String test = split.substring(0,i)+"Descriptor"+split.substring(i+1,split.length()); // replace current space with 'Descriptor' ..
+          test = test.replaceAll("\\s",""); // .. and remove other spaces
+          String className = "org.openscience.cdk.qsar.descriptors.molecular." + test;
+          try {
+              Class.forName(className);
+              return className;
+          } catch (ClassNotFoundException e) {}
         }
-	System.err.println("Descriptor not found: "+descriptorName);
-	System.exit(1);
-	return null;
+      }
+      System.err.println("Descriptor not found: "+descriptorName);
+      System.exit(1);
+      return null;
     }
 
     /** inserts space in between camel words */

data/java/Rakefile

@@ -1,7 +1,7 @@
 # Java class, classpath
 java_classes = [
-  ["CdkDescriptors", "cdk-1.4.19.jar"],
-  ["CdkDescriptorInfo", "cdk-1.4.19.jar"],
+  ["CdkDescriptors", "cdk-2.0-SNAPSHOT.jar"],
+  ["CdkDescriptorInfo", "cdk-2.0-SNAPSHOT.jar"],
   ["JoelibDescriptors", "joelib2.jar:."],
   ["JoelibDescriptorInfo", "joelib2.jar:."],
 ]
@@ -10,6 +10,6 @@ task :default => java_classes.collect{|c| "#{c.first}.class"}
 
 java_classes.each do |c|
   file "#{c.first}.class" => "#{c.first}.java" do
-    puts `javac -classpath #{c.last} #{c.first}.java`
+    puts `javac -Xlint:deprecation -classpath #{c.last} #{c.first}.java`
   end
 end

data/lazar.gemspec

@@ -18,11 +18,10 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   # specify any dependencies here; for example:
-  s.add_runtime_dependency 'bundler', '~> 1.11'
-  s.add_runtime_dependency 'rest-client', '~> 1.8'
-  s.add_runtime_dependency 'nokogiri', '~> 1.6'
-  s.add_runtime_dependency 'rserve-client', '~> 0.3'
-  s.add_runtime_dependency 'mongoid', '~> 5.0'
-  s.add_runtime_dependency 'openbabel', '~> 2.3', '>= 2.3.2.2'
-
+  s.add_runtime_dependency 'bundler'
+  s.add_runtime_dependency 'rest-client'
+  s.add_runtime_dependency 'nokogiri'
+  s.add_runtime_dependency 'rserve-client'
+  s.add_runtime_dependency 'mongoid'
+  s.add_runtime_dependency 'openbabel'
 end

data/lib/algorithm.rb

@@ -2,18 +2,9 @@ module OpenTox
 
   module Algorithm 
 
-    # Generic method to execute algorithms
-    # Algorithms should:
-    #   - accept a Compound, an Array of Compounds or a Dataset as first argument
-    #   - optional parameters as second argument
-    #   - return an object corresponding to the input type as result (eg. Compound -> value, Array of Compounds -> Array of values, Dataset -> Dataset with values
-    # @param [OpenTox::Compound,Array,OpenTox::Dataset] Input object
-    # @param [Hash] Algorithm parameters
-    # @return Algorithm result
-    def self.run algorithm, object, parameters=nil
-      bad_request_error "Cannot run '#{algorithm}' algorithm. Please provide an OpenTox::Algorithm." unless algorithm =~ /^OpenTox::Algorithm/
+    def self.run algorithm, parameters=nil
       klass,method = algorithm.split('.')
-      parameters.nil? ?  Object.const_get(klass).send(method,object) : Object.const_get(klass).send(method,object, parameters)
+      Object.const_get(klass).send(method,parameters) 
     end
 
   end

data/lib/caret.rb

@@ -0,0 +1,96 @@
+module OpenTox
+  module Algorithm
+    
+    class Caret
+      # model list: https://topepo.github.io/caret/modelList.html
+
+      def self.create_model_and_predict dependent_variables:, independent_variables:, weights:, method:, query_variables:
+        remove = []
+        # remove independent_variables with single values
+        independent_variables.each_with_index { |values,i| remove << i if values.uniq.size == 1}
+        remove.sort.reverse.each do |i|
+          independent_variables.delete_at i
+          query_variables.delete_at i
+        end
+        if independent_variables.flatten.uniq == ["NA"] or independent_variables.flatten.uniq == [] 
+          prediction = Algorithm::Regression::weighted_average dependent_variables:dependent_variables, weights:weights
+          prediction[:warning] = "No variables for regression model. Using weighted average of similar substances."
+        elsif
+          dependent_variables.size < 3
+          prediction = Algorithm::Regression::weighted_average dependent_variables:dependent_variables, weights:weights
+          prediction[:warning] = "Insufficient number of neighbors (#{dependent_variables.size}) for regression model. Using weighted average of similar substances."
+
+        else
+          dependent_variables.each_with_index do |v,i| 
+            dependent_variables[i] = to_r(v)
+          end
+          independent_variables.each_with_index do |c,i| 
+            c.each_with_index do |v,j|
+              independent_variables[i][j] = to_r(v)
+            end
+          end
+          query_variables.each_with_index do |v,i| 
+            query_variables[i] = to_r(v)
+          end
+          begin
+            R.assign "weights", weights
+            r_data_frame = "data.frame(#{([dependent_variables]+independent_variables).collect{|r| "c(#{r.join(',')})"}.join(', ')})"
+            R.eval "data <- #{r_data_frame}"
+            R.assign "features", (0..independent_variables.size-1).to_a
+            R.eval "names(data) <- append(c('activities'),features)" #
+            R.eval "model <- train(activities ~ ., data = data, method = '#{method}', na.action = na.pass, allowParallel=TRUE)"
+          rescue => e
+            $logger.debug "R caret model creation error for:"
+            $logger.debug dependent_variables
+            $logger.debug independent_variables
+            prediction = Algorithm::Regression::weighted_average dependent_variables:dependent_variables, weights:weights
+            prediction[:warning] = "R caret model creation error. Using weighted average of similar substances."
+            return prediction
+          end
+          begin
+            R.eval "query <- data.frame(rbind(c(#{query_variables.join ','})))"
+            R.eval "names(query) <- features" 
+            R.eval "prediction <- predict(model,query)"
+            value = R.eval("prediction").to_f
+            rmse = R.eval("getTrainPerf(model)$TrainRMSE").to_f
+            r_squared = R.eval("getTrainPerf(model)$TrainRsquared").to_f
+            prediction_interval = value-1.96*rmse, value+1.96*rmse
+            prediction = {
+              :value => value,
+              :rmse => rmse,
+              :r_squared => r_squared,
+              :prediction_interval => prediction_interval
+            }
+          rescue => e
+            $logger.debug "R caret prediction error for:"
+            $logger.debug self.inspect
+            prediction = Algorithm::Regression::weighted_average dependent_variables:dependent_variables, weights:weights
+            prediction[:warning] = "R caret prediction error. Using weighted average of similar substances"
+            return prediction
+          end
+          if prediction.nil? or prediction[:value].nil?
+            prediction = Algorithm::Regression::weighted_average dependent_variables:dependent_variables, weights:weights
+            prediction[:warning] = "Could not create local caret model. Using weighted average of similar substances."
+          end
+        end
+        prediction
+
+      end
+
+      # call caret methods dynamically, e.g. Caret.pls
+      def self.method_missing(sym, *args, &block)
+        args.first[:method] = sym.to_s
+        self.create_model_and_predict args.first
+      end
+
+      def self.to_r v
+        return "F" if v == false
+        return "T" if v == true
+        return nil if v.is_a? Float and v.nan?
+        v
+      end
+
+    end
+  end
+end
+

data/lib/classification.rb

@@ -3,32 +3,24 @@ module OpenTox
     
     class Classification
 
-      def self.weighted_majority_vote compound, params
-        neighbors = params[:neighbors]
-        weighted_sum = {}
-        sim_sum = 0.0
-        confidence = 0.0
-        neighbors.each do |row|
-          sim = row["tanimoto"]
-          row["features"][params[:prediction_feature_id].to_s].each do |act|
-            weighted_sum[act] ||= 0
-            weighted_sum[act] += sim
-          end
+      def self.weighted_majority_vote dependent_variables:, independent_variables:nil, weights:, query_variables:
+        class_weights = {}
+        dependent_variables.each_with_index do |v,i|
+          class_weights[v] ||= []
+          class_weights[v] << weights[i] unless v.nil?
         end
-        case weighted_sum.size
-        when 1
-          return {:value => weighted_sum.keys.first, :confidence => weighted_sum.values.first/neighbors.size.abs}
-        when 2
-          sim_sum = weighted_sum[weighted_sum.keys[0]]
-          sim_sum -= weighted_sum[weighted_sum.keys[1]]
-          sim_sum > 0 ? prediction = weighted_sum.keys[0] : prediction = weighted_sum.keys[1] 
-          confidence = (sim_sum/neighbors.size).abs 
-          return {:value => prediction,:confidence => confidence}
-        else
-          bad_request_error "Cannot predict more than 2 classes, multinomial classifications is not yet implemented. Received classes were: '#{weighted.sum.keys}'"
+        probabilities = {}
+        class_weights.each do |a,w|
+          probabilities[a] = w.sum/weights.sum
         end
+        probabilities = probabilities.collect{|a,p| [a,weights.max*p]}.to_h
+        p_max = probabilities.collect{|a,p| p}.max
+        prediction = probabilities.key(p_max)
+        {:value => prediction,:probabilities => probabilities}
       end
+
     end
+
   end
 end
 

data/lib/compound.rb

@@ -1,11 +1,9 @@
-CACTUS_URI="http://cactus.nci.nih.gov/chemical/structure/"
+CACTUS_URI="https://cactus.nci.nih.gov/chemical/structure/"
 
 module OpenTox
 
-  class Compound
+  class Compound < Substance
     require_relative "unique_descriptors.rb"
-    include OpenTox
-
     DEFAULT_FINGERPRINT = "MP2D"
 
     field :inchi, type: String
@@ -19,9 +17,6 @@ module OpenTox
     field :sdf_id, type: BSON::ObjectId
     field :fingerprints, type: Hash, default: {}
     field :default_fingerprint_size, type: Integer
-    field :physchem_descriptors, type: Hash, default: {}
-    field :dataset_ids, type: Array, default: []
-    field :features, type: Hash, default: {}
 
     index({smiles: 1}, {unique: true})
 
@@ -80,9 +75,8 @@ module OpenTox
       fingerprints[type]
     end
 
-    def physchem descriptors=PhysChem.openbabel_descriptors
-      # TODO: speedup java descriptors
-      calculated_ids = physchem_descriptors.keys
+    def calculate_properties descriptors=PhysChem::OPENBABEL
+      calculated_ids = properties.keys
       # BSON::ObjectId instances are not allowed as keys in a BSON document.
       new_ids = descriptors.collect{|d| d.id.to_s} - calculated_ids
       descs = {}
@@ -95,11 +89,11 @@ module OpenTox
       # avoid recalculating Cdk features with multiple values
       descs.keys.uniq.each do |k|
         descs[k].send(k[0].downcase,k[1],self).each do |n,v|
-          physchem_descriptors[algos[n].id.to_s] = v # BSON::ObjectId instances are not allowed as keys in a BSON document.
+          properties[algos[n].id.to_s] = v # BSON::ObjectId instances are not allowed as keys in a BSON document.
         end
       end
       save
-      physchem_descriptors.select{|id,v| descriptors.collect{|d| d.id.to_s}.include? id}
+      descriptors.collect{|d| properties[d.id.to_s]}
     end
 
     def smarts_match smarts, count=false
@@ -142,9 +136,6 @@ module OpenTox
     # @param inchi [String] smiles InChI string
     # @return [OpenTox::Compound] Compound
     def self.from_inchi inchi
-      # Temporary workaround for OpenBabels Inchi bug
-      # http://sourceforge.net/p/openbabel/bugs/957/
-      # bug has not been fixed in latest git/development version
       #smiles = `echo "#{inchi}" | "#{File.join(File.dirname(__FILE__),"..","openbabel","bin","babel")}" -iinchi - -ocan`.chomp.strip
       smiles = obconversion(inchi,"inchi","can")
       if smiles.empty?
@@ -246,7 +237,7 @@ module OpenTox
 
     # @return [String] PubChem Compound Identifier (CID), derieved via restcall to pubchem
     def cid
-      pug_uri = "http://pubchem.ncbi.nlm.nih.gov/rest/pug/"
+      pug_uri = "https://pubchem.ncbi.nlm.nih.gov/rest/pug/"
       update(:cid => RestClientWrapper.post(File.join(pug_uri, "compound", "inchi", "cids", "TXT"),{:inchi => inchi}).strip) unless self["cid"] 
       self["cid"]
     end
@@ -254,70 +245,13 @@ module OpenTox
     # @return [String] ChEMBL database compound id, derieved via restcall to chembl
     def chemblid
       # https://www.ebi.ac.uk/chembldb/ws#individualCompoundByInChiKey
-      uri = "http://www.ebi.ac.uk/chemblws/compounds/smiles/#{smiles}.json"
+      uri = "https://www.ebi.ac.uk/chemblws/compounds/smiles/#{smiles}.json"
       update(:chemblid => JSON.parse(RestClientWrapper.get(uri))["compounds"].first["chemblId"]) unless self["chemblid"] 
       self["chemblid"]
     end
 
-    def fingerprint_count_neighbors params
-      # TODO fix
-      neighbors = []
-      query_fingerprint = self.fingerprint params[:type]
-      training_dataset = Dataset.find(params[:training_dataset_id]).compounds.each do |compound|
-        unless self == compound
-          candidate_fingerprint = compound.fingerprint params[:type]
-          features = (query_fingerprint + candidate_fingerprint).uniq
-          min_sum = 0
-          max_sum = 0
-          features.each do |f|
-            min,max = [query_fingerprint.count(f),candidate_fingerprint.count(f)].minmax
-            min_sum += min
-            max_sum += max
-          end
-          max_sum == 0 ? sim = 0 : sim = min_sum/max_sum.to_f
-          neighbors << [compound.id, sim] if sim and sim >= params[:min_sim]
-        end
-      end
-      neighbors.sort{|a,b| b.last <=> a.last}
-    end
-
-    def fingerprint_neighbors params
-      bad_request_error "Incorrect parameters '#{params}' for Compound#fingerprint_neighbors. Please provide :type, :training_dataset_id, :min_sim." unless params[:type] and params[:training_dataset_id] and params[:min_sim]
-      neighbors = []
-      if params[:type] == DEFAULT_FINGERPRINT
-        neighbors = db_neighbors params
-      else 
-        query_fingerprint = self.fingerprint params[:type]
-        training_dataset = Dataset.find(params[:training_dataset_id])
-        prediction_feature = training_dataset.features.first
-        training_dataset.compounds.each do |compound|
-          candidate_fingerprint = compound.fingerprint params[:type]
-          sim = (query_fingerprint & candidate_fingerprint).size/(query_fingerprint | candidate_fingerprint).size.to_f
-          feature_values = training_dataset.values(compound,prediction_feature)
-          neighbors << {"_id" => compound.id, "features" => {prediction_feature.id.to_s => feature_values}, "tanimoto" => sim} if sim >= params[:min_sim]
-        end
-        neighbors.sort!{|a,b| b["tanimoto"] <=> a["tanimoto"]}
-      end
-      neighbors
-    end
-
-    def physchem_neighbors params
-      feature_dataset = Dataset.find params[:feature_dataset_id]
-      query_fingerprint = Algorithm.run params[:feature_calculation_algorithm], self, params[:descriptors]
-      neighbors = []
-      feature_dataset.data_entries.each_with_index do |candidate_fingerprint, i|
-        # TODO implement pearson and cosine similarity separatly
-        R.assign "x", query_fingerprint
-        R.assign "y", candidate_fingerprint
-        sim = R.eval("x %*% y / sqrt(x%*%x * y%*%y)").to_ruby.first
-        if sim >= params[:min_sim]
-          neighbors << [feature_dataset.compound_ids[i],sim] # use compound_ids, instantiation of Compounds is too time consuming
-        end
-      end
-      neighbors
-    end
-
-    def db_neighbors params
+    def db_neighbors min_sim: 0.1, dataset_id:
+      #p fingerprints[DEFAULT_FINGERPRINT]
       # from http://blog.matt-swain.com/post/87093745652/chemical-similarity-search-in-mongodb
 
       #qn = default_fingerprint_size
@@ -329,31 +263,31 @@ module OpenTox
         #{'$match': {'mfp.count': {'$gte': qmin, '$lte': qmax}, 'mfp.bits': {'$in': reqbits}}},
         #{'$match' =>  {'_id' => {'$ne' => self.id}}}, # remove self
         {'$project' => {
-          'tanimoto' => {'$let' => {
+          'similarity' => {'$let' => {
             'vars' => {'common' => {'$size' => {'$setIntersection' => ["$fingerprints.#{DEFAULT_FINGERPRINT}", fingerprints[DEFAULT_FINGERPRINT]]}}},
-            #'vars' => {'common' => {'$size' => {'$setIntersection' => ["$default_fingerprint", default_fingerprint]}}},
             'in' => {'$divide' => ['$$common', {'$subtract' => [{'$add' => [default_fingerprint_size, '$default_fingerprint_size']}, '$$common']}]}
           }},
           '_id' => 1,
-          'features' => 1,
+          #'measurements' => 1,
           'dataset_ids' => 1
         }},
-        {'$match' =>  {'tanimoto' => {'$gte' => params[:min_sim]}}},
-        {'$sort' => {'tanimoto' => -1}}
+        {'$match' =>  {'similarity' => {'$gte' => min_sim}}},
+        {'$sort' => {'similarity' => -1}}
       ]
-      
-      $mongo["compounds"].aggregate(aggregate).select{|r| r["dataset_ids"].include? params[:training_dataset_id]}
+
+      # TODO move into aggregate pipeline, see http://stackoverflow.com/questions/30537317/mongodb-aggregation-match-if-value-in-array
+      $mongo["substances"].aggregate(aggregate).select{|r| r["dataset_ids"].include? dataset_id}
         
     end
     
-    # Convert mg to mmol
+    # Convert mmol to mg
     # @return [Float] value in mg
     def mmol_to_mg mmol
       mmol.to_f*molecular_weight
     end
 
-    # Convert mmol to mg
-    # @return [Float] value in mg
+    # Convert mg to mmol
+    # @return [Float] value in mmol
     def mg_to_mmol mg
       mg.to_f/molecular_weight
     end
@@ -362,7 +296,7 @@ module OpenTox
     # @return [Float] molecular weight
     def molecular_weight
       mw_feature = PhysChem.find_or_create_by(:name => "Openbabel.MW")
-      physchem([mw_feature])[mw_feature.id.to_s]
+      calculate_properties([mw_feature]).first
     end
 
     private