lazar 0.0.7 → 0.0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 56f51ab78b66037e55ff41d7515b0c4bc3876481
-  data.tar.gz: 893b5f4827406df36ff6abc186767889e4b2cb6c
+  metadata.gz: 1b22cad0ba1ecef02ff4af283796fcb36cbe758f
+  data.tar.gz: 49bd9a98d7c24ff2b7d1442d58d0b775aaf62e74
 SHA512:
-  metadata.gz: b0d402841c42990b7d2a3d8efcbb9c3c7e1839939ad61774a906d289d5a0c7a33277833827175eb006d922f13da24d7c489aaba5e9c25b967dc6ea18964d9333
-  data.tar.gz: 2242413832ffe15e2ec4bcbb8bf33a0fe126e365d163fe55c804bcd6dc3741ae6f0058dd3c39b7a70121a82e81586b190787dcce96fc504bc1e5aae32af3ec10
+  metadata.gz: 96bd32e2b21abfb827a5cfa10ee520a1c06158876d4fb6238da63b79a785137fcc587aa78f40c8ec03b708e83a520c0cd0192c0795f4df34dbf05ebc21677a3c
+  data.tar.gz: c54ea1804b359da06a32b6c4a8314cc329cd173aa8defbdd7adc63c46230c4c75cf7489beecdb5d3290b6892352778e996cf6828d4ac576ef4082e1ef6c93a46

data/.gitignore

@@ -1,5 +1,7 @@
 last-utils
 libfminer
+openbabel
+fminer_debug.txt
 test/fminer_debug.txt
 Gemfile.lock
 *.gem
@@ -8,3 +10,4 @@ pkg/*
 *~
 .yardoc/
 doc/
+lazar.log

data/README.md

@@ -8,7 +8,7 @@ Dependencies
 
   lazar depends on a couple of external programs and libraries. On Debian 7 "Wheezy" systems you can install them with
 
-   `sudo apt-get install build-essential ruby ruby-dev git cmake swig r-base r-base-dev r-cran-rserve openjdk-7-jre libgsl0-dev libxml2-dev zlib1g-dev libcairo2-dev`
+   `sudo apt-get install build-essential ruby ruby-dev git cmake swig r-base r-base-dev openjdk-7-jre libgsl0-dev libxml2-dev zlib1g-dev libcairo2-dev`
   
   You will also need at least mongodb version 3.0, but Debian "Wheezy" provides version 2.4. Please follow the instructions at http://docs.mongodb.org/manual/tutorial/install-mongodb-on-debian/:
 
@@ -30,6 +30,7 @@ Installation
   git clone https://github.com/opentox/lazar.git
   cd lazar
   ruby ext/lazar/extconf.rb
+  sudo Rscript ext/lazar/rinstall.R
   bundle install
   ```
 

data/VERSION

@@ -1 +1 @@
-0.0.7
+0.0.9

data/ext/lazar/extconf.rb

@@ -1,88 +1,27 @@
 require 'fileutils'
 require 'rbconfig'
+require 'mkmf'
 
 main_dir = File.expand_path(File.join(File.dirname(__FILE__),"..",".."))
 
-# install OpenBabel
-
-
-openbabel_version = "2.3.2"
-
-openbabel_dir = File.join main_dir, "openbabel"
-src_dir = openbabel_dir #File.join openbabel_dir, "openbabel-#{openbabel_version}"
-build_dir = File.join src_dir, "build"
-install_dir = openbabel_dir 
-install_lib_dir = File.join install_dir, "lib"
-lib_dir = File.join openbabel_dir, "lib", "openbabel"
-ruby_src_dir = File.join src_dir, "scripts", "ruby"
-
-begin
-  nr_processors = `grep processor /proc/cpuinfo | wc -l` # speed up compilation, Linux only
-rescue
-  nr_processors = 1
+# check for required programs
+programs = ["R","Rscript","mongod","java","getconf"]
+programs.each do |program|
+  abort "Please install #{program} on your system." unless find_executable program
 end
 
-FileUtils.mkdir_p openbabel_dir
-Dir.chdir main_dir do
-  FileUtils.rm_rf src_dir
-  puts "Downloading OpenBabel sources"
-  system "git clone https://github.com/openbabel/openbabel.git"
-end
+abort "Please install Rserve on your system. Execute 'install.packages('Rserve')' in a R console running as root ('sudo R')."  unless `R CMD Rserve --version`.match(/^Rserve/)
 
-FileUtils.mkdir_p build_dir
-FileUtils.mkdir_p install_dir
-Dir.chdir build_dir do
-  puts "Configuring OpenBabel"
-  cmake = "cmake #{src_dir} -DCMAKE_INSTALL_PREFIX=#{install_dir} -DBUILD_GUI=OFF -DENABLE_TESTS=OFF -DRUN_SWIG=ON -DRUBY_BINDINGS=ON"
-  # set rpath for local installations
-  # http://www.cmake.org/Wiki/CMake_RPATH_handling
-  # http://vtk.1045678.n5.nabble.com/How-to-force-cmake-not-to-remove-install-rpath-td5721193.html
-  cmake += " -DCMAKE_INSTALL_RPATH:STRING=\"#{install_lib_dir}\"" 
-  system cmake
-end
+# install R packages
+r_dir = File.join main_dir, "R"
+FileUtils.mkdir_p r_dir
+FileUtils.mkdir_p File.join(main_dir,"bin") # for Rserve binary
+rinstall = File.expand_path(File.join(File.dirname(__FILE__),"rinstall.R"))
+puts `Rscript --vanilla #{rinstall} #{r_dir}`
 
-# local installation in gem directory
-Dir.chdir build_dir do
-  puts "Compiling OpenBabel sources."
-  system "make -j#{nr_processors}"
-  system "make install"
-  ENV["PKG_CONFIG_PATH"] = File.dirname(File.expand_path(Dir["#{install_dir}/**/openbabel*pc"].first))
+# create a fake Makefile
+File.open(File.join(File.dirname(__FILE__),"Makefile"),"w+") do |makefile|
+  makefile.puts "all:\n\ttrue\n\ninstall:\n\ttrue\n"
 end
 
-ob_include= File.expand_path File.join(File.dirname(__FILE__),"../../openbabel/include/openbabel-2.0")
-ob_lib= File.expand_path File.join(File.dirname(__FILE__),"../../openbabel/lib")
-
-# compile ruby bindings
-=begin
-puts "Compiling and installing OpenBabel Ruby bindings."
-Dir.chdir ruby_src_dir do
-  # fix rpath
-  system "sed -i 's|with_ldflags.*$|with_ldflags(\"#\$LDFLAGS -dynamic -Wl,-rpath,#{install_lib_dir}\") do|' #{File.join(ruby_src_dir,'extconf.rb')}"
-  system "#{RbConfig.ruby} extconf.rb --with-openbabel-include=#{ob_include} --with-openbabel-lib=#{ob_lib}"
-  system "make -j#{nr_processors}"
-end
-=end
-
-# install fminer
-fminer_dir = File.join main_dir, "libfminer"
-system "git clone git://github.com/amaunz/fminer2.git #{fminer_dir}"
-
-["libbbrc","liblast"].each do |lib|
-  FileUtils.cd File.join(fminer_dir,lib)
-  system "sed -i 's,^INCLUDE_OB.*,INCLUDE_OB\ =\ #{ob_include},g' Makefile" 
-  system "sed -i 's,^LDFLAGS_OB.*,LDFLAGS_OB\ =\ #{ob_lib},g' Makefile"
-  system "sed -i 's,^INCLUDE_RB.*,INCLUDE_RB\ =\ #{RbConfig::CONFIG['rubyhdrdir']},g' Makefile" 
-  # TODO fix in fminer Makefile
-  system "sed -i 's,-g, -g -I #{RbConfig::CONFIG['rubyhdrdir']} -I #{RbConfig::CONFIG['rubyarchhdrdir']} -I,' Makefile" # fix include path (CH)
-  system "sed -i '74s/$(CC)/$(CC) -Wl,-rpath,#{ob_lib.gsub('/','\/')} -L/' Makefile" # fix library path (CH)
-  system "make ruby"
-end
-
-# install last-utils
-FileUtils.cd main_dir
-system "git clone git://github.com/amaunz/last-utils.git"
-FileUtils.cd File.join(main_dir,"last-utils")
-`sed -i '8s/"openbabel", //' lu.rb`
-
-# install R packagemain_dir
 $makefile_created = true

data/ext/lazar/rinstall.R

@@ -0,0 +1,9 @@
+libdir = commandArgs(trailingOnly=TRUE)[1]
+# chooseCRANmirror(ind=19); does not have any impact on selected server
+#args=paste0("--prefix=",libdir,"/..")
+#install.packages("Rserve",lib=libdir,configure.args=args)
+install.packages("gridExtra",lib=libdir);
+install.packages("ggplot2",lib=libdir);
+install.packages("pls",lib=libdir);
+install.packages("caret",lib=libdir);
+install.packages("doMC",lib=libdir);

data/lazar.gemspec

@@ -9,20 +9,20 @@ Gem::Specification.new do |s|
   s.homepage    = "http://github.com/opentox/lazar"
   s.summary     = %q{Lazar framework}
   s.description = %q{Libraries for lazy structure-activity relationships and read-across.}
-  s.license     = 'GPL-3'
+  s.license     = 'GPL-3.0'
 
   s.rubyforge_project = "lazar"
-
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.extensions    = %w[ext/lazar/extconf.rb]
   s.require_paths = ["lib"]
 
   # specify any dependencies here; for example:
-  s.add_runtime_dependency "bundler"
-  s.add_runtime_dependency "rest-client"
-  s.add_runtime_dependency 'nokogiri'
-  s.add_runtime_dependency 'rserve-client'
-  s.add_runtime_dependency "mongoid", '~> 5.0beta'  
+  s.add_runtime_dependency "bundler", "~> 1.11"
+  s.add_runtime_dependency "rest-client", "~> 1.8"
+  s.add_runtime_dependency 'nokogiri', "~> 1.6"
+  s.add_runtime_dependency 'rserve-client', "~> 0.3"
+  s.add_runtime_dependency "mongoid", "~> 5.0"
+  s.add_runtime_dependency 'openbabel> 2.3.2.2', '~> 0'
 
 end

data/lib/classification.rb

@@ -3,13 +3,14 @@ module OpenTox
     
     class Classification
 
-      def self.weighted_majority_vote neighbors
-        return {:value => nil,:confidence => nil,:warning => "Cound not find similar compounds."} if neighbors.empty?
+      def self.weighted_majority_vote compound, params
+        neighbors = params[:neighbors]
         weighted_sum = {}
         sim_sum = 0.0
+        confidence = 0.0
         neighbors.each do |row|
-          n,sim,acts = row
-          acts.each do |act|
+          sim = row["tanimoto"]
+          row["features"][params[:prediction_feature_id].to_s].each do |act|
             weighted_sum[act] ||= 0
             weighted_sum[act] += sim
           end
@@ -27,81 +28,7 @@ module OpenTox
           bad_request_error "Cannot predict more than 2 classes, multinomial classifications is not yet implemented. Received classes were: '#{weighted.sum.keys}'"
         end
       end
-
-      # Classification with majority vote from neighbors weighted by similarity
-      # @param [Hash] params Keys `:activities, :sims, :value_map` are required
-      # @return [Numeric] A prediction value.
-      def self.fminer_weighted_majority_vote neighbors, training_dataset
-
-        neighbor_contribution = 0.0
-        confidence_sum = 0.0
-
-        $logger.debug "Weighted Majority Vote Classification."
-
-        values = neighbors.collect{|n| n[2]}.uniq
-        neighbors.each do |neighbor|
-          i = training_dataset.compound_ids.index n.id
-          neighbor_weight = neighbor[1]
-          activity = values.index(neighbor[2]) + 1 # map values to integers > 1
-          neighbor_contribution += activity * neighbor_weight
-          if values.size == 2 # AM: provide compat to binary classification: 1=>false 2=>true
-            case activity
-            when 1
-              confidence_sum -= neighbor_weight
-            when 2
-              confidence_sum += neighbor_weight
-            end
-          else
-            confidence_sum += neighbor_weight
-          end
-        end
-        if values.size == 2 
-          if confidence_sum >= 0.0
-            prediction = values[1]
-          elsif confidence_sum < 0.0
-            prediction = values[0] 
-          end
-        elsif values.size == 1 # all neighbors have the same value
-          prediction = values[0] 
-        else 
-          prediction = (neighbor_contribution/confidence_sum).round  # AM: new multinomial prediction
-        end 
-
-        confidence = (confidence_sum/neighbors.size).abs 
-        {:value => prediction, :confidence => confidence.abs}
-      end
-
-      # Local support vector regression from neighbors 
-      # @param [Hash] params Keys `:props, :activities, :sims, :min_train_performance` are required
-      # @return [Numeric] A prediction value.
-      def self.local_svm_classification(params)
-
-        confidence = 0.0
-        prediction = nil
-
-        $logger.debug "Local SVM."
-        if params[:activities].size>0
-          if params[:props]
-            n_prop = params[:props][0].collect.to_a
-            q_prop = params[:props][1].collect.to_a
-            props = [ n_prop, q_prop ]
-          end
-          activities = params[:activities].collect.to_a
-          activities = activities.collect{|v| "Val" + v.to_s} # Convert to string for R to recognize classification
-          prediction = local_svm_prop( props, activities, params[:min_train_performance]) # params[:props].nil? signals non-prop setting
-          prediction = prediction.sub(/Val/,"") if prediction # Convert back
-          confidence = 0.0 if prediction.nil?
-          #$logger.debug "Prediction: '" + prediction.to_s + "' ('#{prediction.class}')."
-          confidence = get_confidence({:sims => params[:sims][1], :activities => params[:activities]})
-        end
-        {:value => prediction, :confidence => confidence}
-
-      end
-
-
-
     end
-
   end
 end
 

data/lib/compound.rb

@@ -1,43 +1,122 @@
-# TODO: check
-# *** Open Babel Error  in ParseFile
-#    Could not find contribution data file.
-
 CACTUS_URI="http://cactus.nci.nih.gov/chemical/structure/"
 
 module OpenTox
 
   class Compound
+    require_relative "unique_descriptors.rb"
     include OpenTox
 
+    DEFAULT_FINGERPRINT = "MP2D"
+
     field :inchi, type: String
     field :smiles, type: String
     field :inchikey, type: String
     field :names, type: Array
-    field :warning, type: String
     field :cid, type: String
     field :chemblid, type: String
     field :png_id, type: BSON::ObjectId
     field :svg_id, type: BSON::ObjectId
     field :sdf_id, type: BSON::ObjectId
-    field :fp4, type: Array
-    field :fp4_size, type: Integer
+    field :fingerprints, type: Hash, default: {}
+    field :default_fingerprint_size, type: Integer
+    field :physchem_descriptors, type: Hash, default: {}
+    field :dataset_ids, type: Array, default: []
+    field :features, type: Hash, default: {}
+
+    index({smiles: 1}, {unique: true})
 
     # Overwrites standard Mongoid method to create fingerprints before database insertion
     def self.find_or_create_by params
       compound = self.find_or_initialize_by params
-      unless compound.fp4 and !compound.fp4.empty?
-        compound.fp4_size = 0
-        compound.fp4 = []
-        fingerprint = FingerprintSmarts.fingerprint
-        Algorithm::Descriptor.smarts_match(compound, fingerprint).each_with_index do |m,i|
-          if m > 0
-            compound.fp4 << fingerprint[i].id
-            compound.fp4_size += 1
+      compound.default_fingerprint_size = compound.fingerprint(DEFAULT_FINGERPRINT).size
+      compound.save
+      compound
+    end
+
+    def fingerprint type=DEFAULT_FINGERPRINT
+      unless fingerprints[type]
+        return [] unless self.smiles
+        #http://openbabel.org/docs/dev/FileFormats/MolPrint2D_format.html#molprint2d-format
+        if type == "MP2D"
+          fp = obconversion(smiles,"smi","mpd").strip.split("\t")
+          name = fp.shift # remove Title
+          fingerprints[type] = fp.uniq # no fingerprint counts
+        #http://openbabel.org/docs/dev/FileFormats/Multilevel_Neighborhoods_of_Atoms_(MNA).html
+        elsif type== "MNA"
+          level = 2 # TODO: level as parameter, evaluate level 1, see paper
+          fp = obconversion(smiles,"smi","mna","xL\"#{level}\"").split("\n")
+          fp.shift # remove Title
+          fingerprints[type] = fp
+        else # standard fingerprints
+          fp = OpenBabel::OBFingerprint.find_fingerprint(type)
+          obmol = OpenBabel::OBMol.new
+          obconversion = OpenBabel::OBConversion.new
+          obconversion.set_in_format "smi"
+          obconversion.read_string obmol, self.smiles
+          result = OpenBabel::VectorUnsignedInt.new
+          fp.get_fingerprint(obmol,result)
+          # TODO: %ignore *::DescribeBits @ line 163 openbabel/scripts/openbabel-ruby.i
+          #p OpenBabel::OBFingerprint.describe_bits(result)
+          # convert result to a list of the bits that are set
+          # from openbabel/scripts/python/pybel.py line 830
+          # see also http://openbabel.org/docs/dev/UseTheLibrary/Python_Pybel.html#fingerprints
+          result = result.to_a
+          bitsperint = OpenBabel::OBFingerprint.getbitsperint()
+          bits_set = []
+          start = 1
+          result.each do |x|
+            i = start
+            while x > 0 do
+              bits_set << i if (x % 2) == 1
+              x >>= 1
+              i += 1
+            end
+            start += bitsperint
           end
+          fingerprints[type] = bits_set
         end
+        save
+      end
+      fingerprints[type]
+    end
+
+    def physchem descriptors=PhysChem.openbabel_descriptors
+      # TODO: speedup java descriptors
+      calculated_ids = physchem_descriptors.keys
+      # BSON::ObjectId instances are not allowed as keys in a BSON document.
+      new_ids = descriptors.collect{|d| d.id.to_s} - calculated_ids
+      descs = {}
+      algos = {}
+      new_ids.each do |id|
+        descriptor = PhysChem.find id
+        descs[[descriptor.library, descriptor.descriptor]]  = descriptor
+        algos[descriptor.name] = descriptor
+      end
+      # avoid recalculating Cdk features with multiple values
+      descs.keys.uniq.each do |k|
+        descs[k].send(k[0].downcase,k[1],self).each do |n,v|
+          physchem_descriptors[algos[n].id.to_s] = v # BSON::ObjectId instances are not allowed as keys in a BSON document.
+        end
+      end
+      save
+      physchem_descriptors.select{|id,v| descriptors.collect{|d| d.id.to_s}.include? id}
+    end
+
+    def smarts_match smarts, count=false
+      obconversion = OpenBabel::OBConversion.new
+      obmol = OpenBabel::OBMol.new
+      obconversion.set_in_format('smi')
+      obconversion.read_string(obmol,self.smiles)
+      smarts_pattern = OpenBabel::OBSmartsPattern.new
+      smarts.collect do |sma|
+        smarts_pattern.init(sma.smarts)
+        if smarts_pattern.match(obmol)
+          count ? value = smarts_pattern.get_map_list.to_a.size : value = 1
+        else
+          value = 0 
+        end
+        value
       end
-      compound.save
-      compound
     end
 
     # Create a compound from smiles string
@@ -46,11 +125,16 @@ module OpenTox
     # @param [String] smiles Smiles string
     # @return [OpenTox::Compound] Compound
     def self.from_smiles smiles
-      smiles = obconversion(smiles,"smi","can")
+      if smiles.match(/\s/) # spaces seem to confuse obconversion and may lead to invalid smiles
+        $logger.warn "SMILES parsing failed for '#{smiles}'', SMILES string contains whitespaces."
+        return nil
+      end
+      smiles = obconversion(smiles,"smi","can") # test if SMILES is correct and return canonical smiles (for compound comparisons)
       if smiles.empty?
-        Compound.find_or_create_by(:warning => "SMILES parsing failed for '#{smiles}', this may be caused by an incorrect SMILES string.")
+        $logger.warn "SMILES parsing failed for '#{smiles}'', this may be caused by an incorrect SMILES string."
+        return nil
       else
-        Compound.find_or_create_by :smiles => obconversion(smiles,"smi","can")
+        Compound.find_or_create_by :smiles => smiles 
       end
     end
 
@@ -64,7 +148,7 @@ module OpenTox
       #smiles = `echo "#{inchi}" | "#{File.join(File.dirname(__FILE__),"..","openbabel","bin","babel")}" -iinchi - -ocan`.chomp.strip
       smiles = obconversion(inchi,"inchi","can")
       if smiles.empty?
-        Compound.find_or_create_by(:warning => "InChi parsing failed for #{inchi}, this may be caused by an incorrect InChi string or a bug in OpenBabel libraries.")
+        Compound.find_or_create_by(:warnings => ["InChi parsing failed for #{inchi}, this may be caused by an incorrect InChi string or a bug in OpenBabel libraries."])
       else
         Compound.find_or_create_by(:smiles => smiles, :inchi => inchi)
       end
@@ -94,7 +178,7 @@ module OpenTox
 
         result = obconversion(smiles,"smi","inchi")
         #result = `echo "#{self.smiles}" | "#{File.join(File.dirname(__FILE__),"..","openbabel","bin","babel")}" -ismi - -oinchi`.chomp
-        update(:inchi => result.chomp) unless result.empty?
+        update(:inchi => result.chomp) if result and !result.empty?
       end
       self["inchi"]
     end
@@ -131,7 +215,7 @@ module OpenTox
       if self.svg_id.nil?
        svg = obconversion(smiles,"smi","svg")
        file = Mongo::Grid::File.new(svg, :filename => "#{id}.svg", :content_type => "image/svg")
-       update(:image_id => $gridfs.insert_one(file))
+       update(:svg_id => $gridfs.insert_one(file))
       end
       $gridfs.find_one(_id: self.svg_id).data
 
@@ -175,32 +259,111 @@ module OpenTox
       self["chemblid"]
     end
 
-    def neighbors threshold=0.7
+    def fingerprint_count_neighbors params
+      # TODO fix
+      neighbors = []
+      query_fingerprint = self.fingerprint params[:type]
+      training_dataset = Dataset.find(params[:training_dataset_id]).compounds.each do |compound|
+        unless self == compound
+          candidate_fingerprint = compound.fingerprint params[:type]
+          features = (query_fingerprint + candidate_fingerprint).uniq
+          min_sum = 0
+          max_sum = 0
+          features.each do |f|
+            min,max = [query_fingerprint.count(f),candidate_fingerprint.count(f)].minmax
+            min_sum += min
+            max_sum += max
+          end
+          max_sum == 0 ? sim = 0 : sim = min_sum/max_sum.to_f
+          neighbors << [compound.id, sim] if sim and sim >= params[:min_sim]
+        end
+      end
+      neighbors.sort{|a,b| b.last <=> a.last}
+    end
+
+    def fingerprint_neighbors params
+      bad_request_error "Incorrect parameters '#{params}' for Compound#fingerprint_neighbors. Please provide :type, :training_dataset_id, :min_sim." unless params[:type] and params[:training_dataset_id] and params[:min_sim]
+      neighbors = []
+      if params[:type] == DEFAULT_FINGERPRINT
+        neighbors = db_neighbors params
+      else 
+        query_fingerprint = self.fingerprint params[:type]
+        training_dataset = Dataset.find(params[:training_dataset_id])
+        prediction_feature = training_dataset.features.first
+        training_dataset.compounds.each do |compound|
+          candidate_fingerprint = compound.fingerprint params[:type]
+          sim = (query_fingerprint & candidate_fingerprint).size/(query_fingerprint | candidate_fingerprint).size.to_f
+          feature_values = training_dataset.values(compound,prediction_feature)
+          neighbors << {"_id" => compound.id, "features" => {prediction_feature.id.to_s => feature_values}, "tanimoto" => sim} if sim >= params[:min_sim]
+        end
+        neighbors.sort!{|a,b| b["tanimoto"] <=> a["tanimoto"]}
+      end
+      neighbors
+    end
+
+    def physchem_neighbors params
+      feature_dataset = Dataset.find params[:feature_dataset_id]
+      query_fingerprint = Algorithm.run params[:feature_calculation_algorithm], self, params[:descriptors]
+      neighbors = []
+      feature_dataset.data_entries.each_with_index do |candidate_fingerprint, i|
+        # TODO implement pearson and cosine similarity separatly
+        R.assign "x", query_fingerprint
+        R.assign "y", candidate_fingerprint
+        sim = R.eval("x %*% y / sqrt(x%*%x * y%*%y)").to_ruby.first
+        if sim >= params[:min_sim]
+          neighbors << [feature_dataset.compound_ids[i],sim] # use compound_ids, instantiation of Compounds is too time consuming
+        end
+      end
+      neighbors
+    end
+
+    def db_neighbors params
       # from http://blog.matt-swain.com/post/87093745652/chemical-similarity-search-in-mongodb
-      qn = fp4.size
+
+      #qn = default_fingerprint_size
       #qmin = qn * threshold
       #qmax = qn / threshold
       #not sure if it is worth the effort of keeping feature counts up to date (compound deletions, additions, ...)
       #reqbits = [count['_id'] for count in db.mfp_counts.find({'_id': {'$in': qfp}}).sort('count', 1).limit(qn - qmin + 1)]
       aggregate = [
         #{'$match': {'mfp.count': {'$gte': qmin, '$lte': qmax}, 'mfp.bits': {'$in': reqbits}}},
-        {'$match' =>  {'_id' => {'$ne' => self.id}}}, # remove self
+        #{'$match' =>  {'_id' => {'$ne' => self.id}}}, # remove self
         {'$project' => {
           'tanimoto' => {'$let' => {
-            'vars' => {'common' => {'$size' => {'$setIntersection' => ['$fp4', fp4]}}},
-            'in' => {'$divide' => ['$$common', {'$subtract' => [{'$add' => [qn, '$fp4_size']}, '$$common']}]}
+            'vars' => {'common' => {'$size' => {'$setIntersection' => ["$fingerprints.#{DEFAULT_FINGERPRINT}", fingerprints[DEFAULT_FINGERPRINT]]}}},
+            #'vars' => {'common' => {'$size' => {'$setIntersection' => ["$default_fingerprint", default_fingerprint]}}},
+            'in' => {'$divide' => ['$$common', {'$subtract' => [{'$add' => [default_fingerprint_size, '$default_fingerprint_size']}, '$$common']}]}
           }},
-          '_id' => 1
+          '_id' => 1,
+          'features' => 1,
+          'dataset_ids' => 1
         }},
-        {'$match' =>  {'tanimoto' => {'$gte' => threshold}}},
+        {'$match' =>  {'tanimoto' => {'$gte' => params[:min_sim]}}},
         {'$sort' => {'tanimoto' => -1}}
       ]
       
-      $mongo["compounds"].aggregate(aggregate).collect{ |r| [r["_id"], r["tanimoto"]] }
+      $mongo["compounds"].aggregate(aggregate).select{|r| r["dataset_ids"].include? params[:training_dataset_id]}
         
     end
-=begin
-=end
+    
+    # Convert mg to mmol
+    # @return [Float] value in mg
+    def mmol_to_mg mmol
+      mmol.to_f*molecular_weight
+    end
+
+    # Convert mmol to mg
+    # @return [Float] value in mg
+    def mg_to_mmol mg
+      mg.to_f/molecular_weight
+    end
+    
+    # Calculate molecular weight of Compound with OB and store it in object
+    # @return [Float] molecular weight
+    def molecular_weight
+      mw_feature = PhysChem.find_or_create_by(:name => "Openbabel.MW")
+      physchem([mw_feature])[mw_feature.id.to_s]
+    end
 
     private
 
@@ -209,17 +372,12 @@ module OpenTox
       obconversion.set_options(option, OpenBabel::OBConversion::OUTOPTIONS) if option
       obmol = OpenBabel::OBMol.new
       obconversion.set_in_and_out_formats input_format, output_format
+      return nil if identifier.nil?
       obconversion.read_string obmol, identifier
       case output_format
       when /smi|can|inchi/
         obconversion.write_string(obmol).gsub(/\s/,'').chomp
       when /sdf/
-p "SDF conversion"
-        # has no effect
-	#obconversion.add_option("gen3D", OpenBabel::OBConversion::GENOPTIONS)
-        # segfaults with openbabel git master 
-        #OpenBabel::OBOp.find_type("Gen3D").do(obmol) 
-
         # TODO: find disconnected structures
         # strip_salts
         # separate
@@ -231,14 +389,13 @@ p "SDF conversion"
 print sdf
         if sdf.match(/.nan/)
           
-# TODO: fix or eliminate 2d generation
           $logger.warn "3D generation failed for compound #{identifier}, trying to calculate 2D structure"
           obconversion.set_options("gen2D", OpenBabel::OBConversion::GENOPTIONS)
-          #OpenBabel::OBOp.find_type("Gen2D").do(obmol) 
           sdf = obconversion.write_string(obmol)
           if sdf.match(/.nan/)
-            $logger.warn "2D generation failed for compound #{identifier}"
-            sdf = nil
+            $logger.warn "2D generation failed for compound #{identifier}, rendering without coordinates."
+            obconversion.remove_option("gen2D", OpenBabel::OBConversion::GENOPTIONS)
+            sdf = obconversion.write_string(obmol)
           end
         end
         sdf
@@ -248,7 +405,7 @@ print sdf
     end
 
     def obconversion(identifier,input_format,output_format,option=nil)
-      self.class.obconversion(identifier,input_format,output_format,option=nil)
+      self.class.obconversion(identifier,input_format,output_format,option)
     end
   end
 end