opentox-ruby 0.0.1 → 0.0.2

data/lib/task.rb

@@ -2,41 +2,191 @@ $self_task=nil
 
 module OpenTox
 
+  # Class for handling asynchronous tasks
   class Task
+    include OpenTox
+    attr_accessor :http_code, :due_to_time
+    
+    def initialize(uri=nil)
+      super uri
+      @metadata = {
+        DC.title => "",
+        DC.date => "",
+        OT.hasStatus => "Running",
+        OT.percentageCompleted => "0",
+        OT.resultURI => "",
+        DC.creator => "", # not mandatory according to API
+        DC.description => "", # not mandatory according to API
+      }
+    end
+  
+    # Create a new task for the code in the block. Catches halts and exceptions and sets task state to error if necessary. The block has to return the URI of the created resource.
+    # @example
+    #   task = OpenTox::Task.create do
+    #     # this code will be executed as a task
+    #     model = OpenTox::Algorithm.run(params) # this can be time consuming
+    #     model.uri # Important: return URI of the created resource
+    #   end
+    #   task.status # returns "Running", because tasks are forked
+    # @param [String] title Task title
+    # @param [String] creator Task creator
+    # @return [OPenTox::Task] Task 
+    def self.create( title=nil, creator=nil, max_duration=DEFAULT_TASK_MAX_DURATION, description=nil )
+      
+      params = {:title=>title, :creator=>creator, :max_duration=>max_duration, :description=>description }
+      task_uri = RestClientWrapper.post(CONFIG[:services]["opentox-task"], params, nil, false).to_s
+      task = Task.new(task_uri.chomp)
+
+      # measure current memory consumption
+      memory = `free -m|sed -n '2p'`.split
+      free_memory = memory[3].to_i + memory[6].to_i # include cache
+      if free_memory < 20 # require at least 200 M free memory
+        LOGGER.warn "Cannot start task  - not enough memory left (#{free_memory} M free)"
+        task.cancel
+        return task
+        #raise "Insufficient memory to start a new task"
+      end
+
+      cpu_load = `cat /proc/loadavg`.split(/\s+/)[0..2].collect{|c| c.to_f}
+      nr_cpu_cores = `cat /proc/cpuinfo |grep "cpu cores"|cut -d ":" -f2|tr -d " "`.split("\n").collect{|c| c.to_i}.inject{|sum,n| sum+n}
+      if cpu_load[0] > nr_cpu_cores and cpu_load[0] > cpu_load[1] and cpu_load[1] > cpu_load[2] # average CPU load of the last minute is high and CPU load is increasing
+        LOGGER.warn "Cannot start task  - CPU load too high (#{cpu_load.join(", ")})"
+        task.cancel
+        return task
+        #raise "Server too busy to start a new task"
+      end
 
-    # due_to_time is only set in local tasks 
-    TASK_ATTRIBS = [ :uri, :date, :title, :creator, :description, :hasStatus, :percentageCompleted, :resultURI, :due_to_time ]
-    TASK_ATTRIBS.each{ |a| attr_accessor(a) }
-    attr_accessor :http_code
+
+      task_pid = Spork.spork(:logger => LOGGER) do
+        LOGGER.debug "Task #{task.uri} started #{Time.now}"
+        $self_task = task
+        
+        begin
+          result = catch(:halt) do
+            yield task
+          end
+          # catching halt, set task state to error
+          if result && result.is_a?(Array) && result.size==2 && result[0]>202
+            LOGGER.error "task was halted: "+result.inspect
+            task.error(result[1])
+            return
+          end
+          LOGGER.debug "Task #{task.uri} done #{Time.now} -> "+result.to_s
+          task.completed(result)
+        rescue => ex
+          LOGGER.error "task failed: "+ex.message
+          LOGGER.error ": "+ex.backtrace.join("\n")
+          task.error(ex.message)
+        end
+      end  
+      task.pid = task_pid
+      LOGGER.debug "Started task: "+task.uri.to_s
+      task
+    end  
+  
+    # Find a task for querying, status changes
+    # @param [String] uri Task URI
+    # @return [OpenTox::Task] Task object
+    def self.find(uri)
+      task = Task.new(uri)
+      task.load_metadata
+      task
+    end
+
+    # Get a list of all tasks
+    # @param [optional, String] uri URI of task service
+    # @return [text/uri-list] Task URIs
+    def self.all(uri=CONFIG[:services]["opentox-task"])
+      OpenTox.all uri
+    end
+
+    def self.from_yaml(yaml)
+      @metadata = YAML.load(yaml)
+    end
+
+    def self.from_rdfxml(rdfxml)
+      file = Tempfile.open("ot-rdfxml"){|f| f.write(rdfxml)}.path
+      parser = Parser::Owl::Generic.new file
+      @metadata = parser.load_metadata
+    end
+
+    def to_rdfxml
+      s = Serializer::Owl.new
+      s.add_task(@uri,@metadata)
+      s.to_rdfxml
+    end
+
+    def status
+      @metadata[OT.hasStatus]
+    end
+
+    def result_uri
+      @metadata[OT.resultURI]
+    end
+
+    def description
+      @metadata[DC.description]
+    end
     
-    private
-    def initialize(uri)
-      @uri = uri.to_s.strip
+    def cancel
+      RestClientWrapper.put(File.join(@uri,'Cancelled'))
+      load_metadata
+    end
+
+    def completed(uri)
+      RestClientWrapper.put(File.join(@uri,'Completed'),{:resultURI => uri})
+      load_metadata
+    end
+
+    def error(description)
+      RestClientWrapper.put(File.join(@uri,'Error'),{:description => description.to_s[0..2000]})
+      load_metadata
     end
     
-    # create is private now, use OpenTox::Task.as_task
-    def self.create( params )
-      task_uri = RestClientWrapper.post(@@config[:services]["opentox-task"], params, nil, false).to_s
-      Task.find(task_uri.chomp)
+    def pid=(pid)
+      RestClientWrapper.put(File.join(@uri,'pid'), {:pid => pid})
     end
-  
-    public
-    def self.find( uri, accept_header=nil )
-      task = Task.new(uri)
-      task.reload( accept_header )
-      return task
+
+    def running?
+      @metadata[OT.hasStatus] == 'Running'
     end
+
+    def completed?
+      @metadata[OT.hasStatus] == 'Completed'
+    end
+
+    def error?
+      @metadata[OT.hasStatus] == 'Error'
+    end
+
+    def load_metadata
+      if (CONFIG[:yaml_hosts].include?(URI.parse(uri).host))
+        result = RestClientWrapper.get(@uri, {:accept => 'application/x-yaml'}, false)
+        @metadata = YAML.load result.to_s
+        @http_code = result.code
+      else
+        @metadata = Parser::Owl::Generic.new(@uri).load_metadata
+        @http_code = RestClientWrapper.get(uri, {:accept => 'application/rdf+xml'}, false).code
+      end
+    end
+    
+    # create is private now, use OpenTox::Task.as_task
+    #def self.create( params )
+      #task_uri = RestClientWrapper.post(CONFIG[:services]["opentox-task"], params, nil, false).to_s
+      #Task.find(task_uri.chomp)
+    #end
     
+=begin
     def self.from_data(data, content_type, code, base_uri)
       task = Task.new(nil)
       task.http_code = code
       task.reload_from_data(data, content_type, base_uri)
       return task
     end
-    
+
     def reload( accept_header=nil )
       unless accept_header 
-        if (@@config[:yaml_hosts].include?(URI.parse(uri).host))
+        if (CONFIG[:yaml_hosts].include?(URI.parse(uri).host))
           accept_header = "application/x-yaml"
         else
           accept_header = 'application/rdf+xml'
@@ -64,113 +214,46 @@ module OpenTox
       end
       raise "uri is null after loading" unless @uri and @uri.to_s.strip.size>0
     end
-    
-    def cancel
-      RestClientWrapper.put(File.join(@uri,'Cancelled'))
-      reload
-    end
-
-    def completed(uri)
-      RestClientWrapper.put(File.join(@uri,'Completed'),{:resultURI => uri})
-      reload
-    end
-
-    def error(description)
-      RestClientWrapper.put(File.join(@uri,'Error'),{:description => description.to_s[0..2000]})
-      reload
-    end
-    
-    def pid=(pid)
-      RestClientWrapper.put(File.join(@uri,'pid'), {:pid => pid})
-    end
-
-    def running?
-      @hasStatus.to_s == 'Running'
-    end
-
-    def completed?
-      @hasStatus.to_s == 'Completed'
-    end
-
-    def error?
-      @hasStatus.to_s == 'Error'
-    end
+=end
 
     # waits for a task, unless time exceeds or state is no longer running
     def wait_for_completion(dur=0.3)
       
-      if (@uri.match(@@config[:services]["opentox-task"]))
-        due_to_time = (@due_to_time.is_a?(Time) ? @due_to_time : Time.parse(@due_to_time))
-        running_time = due_to_time - (@date.is_a?(Time) ? @date : Time.parse(@date))
-      else
-        # the date of the external task cannot be trusted, offest to local time might be to big
-        due_to_time = Time.new + EXTERNAL_TASK_MAX_DURATION
-        running_time = EXTERNAL_TASK_MAX_DURATION
-      end
+      due_to_time = Time.new + DEFAULT_TASK_MAX_DURATION
       LOGGER.debug "start waiting for task "+@uri.to_s+" at: "+Time.new.to_s+", waiting at least until "+due_to_time.to_s
       
+      load_metadata # for extremely fast tasks
+      check_state
       while self.running?
         sleep dur
-        reload
+        load_metadata 
         check_state
         if (Time.new > due_to_time)
-          raise "max wait time exceeded ("+running_time.to_s+"sec), task: '"+@uri.to_s+"'"
+          raise "max wait time exceeded ("+DEFAULT_TASK_MAX_DURATION.to_s+"sec), task: '"+@uri.to_s+"'"
         end
       end
       
-      LOGGER.debug "Task '"+@hasStatus+"': "+@uri.to_s+", Result: "+@resultURI.to_s
+      LOGGER.debug "Task '"+@metadata[OT.hasStatus]+"': "+@uri.to_s+", Result: "+@metadata[OT.resultURI].to_s
     end
   
+    private
     def check_state
       begin
-        raise "illegal task state, task is completed, resultURI is no URI: '"+@resultURI.to_s+
-            "'" unless @resultURI and Utils.is_uri?(@resultURI) if completed?
+        raise "illegal task state, task is completed, resultURI is no URI: '"+@metadata[OT.resultURI].to_s+
+            "'" unless @metadata[OT.resultURI] and @metadata[OT.resultURI].to_s.uri? if completed?
         
         if @http_code == 202
-          raise "illegal task state, code is 202, but hasStatus is not Running: '"+@hasStatus+"'" unless running?
+          raise "#{@uri}: illegal task state, code is 202, but hasStatus is not Running: '"+@metadata[OT.hasStatus]+"'" unless running?
         elsif @http_code == 201
-          raise "illegal task state, code is 201, but hasStatus is not Completed: '"+@hasStatus+"'" unless completed?
-          raise "illegal task state, code is 201, resultURI is no task-URI: '"+@resultURI.to_s+
-              "'" unless @resultURI and Utils.task_uri?(@resultURI)
+          raise "#{@uri}: illegal task state, code is 201, but hasStatus is not Completed: '"+@metadata[OT.hasStatus]+"'" unless completed?
+          raise "#{@uri}: illegal task state, code is 201, resultURI is no task-URI: '"+@metadata[OT.resultURI].to_s+
+              "'" unless @metadata[OT.resultURI] and @metadata[OT.resultURI].to_s.uri?
         end
       rescue => ex
         RestClientWrapper.raise_uri_error(ex.message, @uri)
       end
     end
-  
-    # returns the task uri
-    # catches halts and exceptions, task state is set to error then
-    def self.as_task( title, creator, max_duration=DEFAULT_TASK_MAX_DURATION, description=nil )
-      #return yield nil
-      
-      params = {:title=>title, :creator=>creator, :max_duration=>max_duration, :description=>description }
-      task = OpenTox::Task.create(params)
-      task_pid = Spork.spork(:logger => LOGGER) do
-        LOGGER.debug "Task #{task.uri} started #{Time.now}"
-        $self_task = task
-        
-        begin
-          result = catch(:halt) do
-            yield task
-          end
-          # catching halt, set task state to error
-          if result && result.is_a?(Array) && result.size==2 && result[0]>202
-            LOGGER.error "task was halted: "+result.inspect
-            task.error(result[1])
-            return
-          end
-          LOGGER.debug "Task #{task.uri} done #{Time.now} -> "+result.to_s
-          task.completed(result)
-        rescue => ex
-          LOGGER.error "task failed: "+ex.message
-          LOGGER.error ": "+ex.backtrace.join("\n")
-          task.error(ex.message)
-        end
-      end  
-      task.pid = task_pid
-      LOGGER.debug "Started task: "+task.uri.to_s
-      task.uri
-    end  
+
   end
 
 end

data/lib/validation.rb

@@ -1,20 +1,70 @@
 module OpenTox
 	class Validation
+    include OpenTox
 
-		attr_accessor :uri
-
-		def initialize(params)
-      @uri = OpenTox::RestClientWrapper.post(File.join(@@config[:services]["opentox-validation"],"/crossvalidation"),params,nil,false)
-		end
+		attr_accessor :report_uri, :qmrf_report_uri
 		
-		def self.crossvalidation(params)
-			params[:uri] = File.join(@@config[:services]['opentox-validation'], "crossvalidation")
+		def self.create_crossvalidation(params)
+			params[:uri] = File.join(CONFIG[:services]['opentox-validation'], "crossvalidation")
       params[:num_folds] = 10 unless params[:num_folds]
 		 	params[:random_seed] = 2 unless params[:random_seed]
 		 	params[:stratified] = false unless params[:stratified]
-			OpenTox::Validation.new(params)
+      uri = OpenTox::RestClientWrapper.post(File.join(CONFIG[:services]["opentox-validation"],"/crossvalidation"),params,nil,false)
+			OpenTox::Validation.new(uri)
 		end
 
+    def create_report
+			@report_uri = RestClientWrapper.post(File.join(CONFIG[:services]["opentox-validation"],"/report/crossvalidation"), :validation_uris => @uri).to_s
+      @report_uri
+    end
+
+    def create_qmrf_report
+      @qmrf_report_uri = RestClientWrapper.post(File.join(CONFIG[:services]["opentox-validation"],"/reach_report/qmrf"), :model_uri => @uri).to_s
+      @qmrf_report_uri
+    end
+
+    def summary(type)
+      v = YAML.load RestClientWrappper.get(File.join(@uri, 'statistics'),:accept => "application/x-yaml").to_s
+
+      case type
+      when "classification"
+        tp=0; tn=0; fp=0; fn=0; n=0
+        v[:classification_statistics][:confusion_matrix][:confusion_matrix_cell].each do |cell|
+          if cell[:confusion_matrix_predicted] == "true" and cell[:confusion_matrix_actual] == "true"
+            tp = cell[:confusion_matrix_value]
+            n += tp
+          elsif cell[:confusion_matrix_predicted] == "false" and cell[:confusion_matrix_actual] == "false"
+            tn = cell[:confusion_matrix_value]
+            n += tn
+          elsif cell[:confusion_matrix_predicted] == "false" and cell[:confusion_matrix_actual] == "true"
+            fn = cell[:confusion_matrix_value]
+            n += fn
+          elsif cell[:confusion_matrix_predicted] == "true" and cell[:confusion_matrix_actual] == "false"
+            fp = cell[:confusion_matrix_value]
+            n += fp
+          end
+        end
+        {
+          :nr_predictions => n,
+          :true_positives => tp,
+          :false_positives => fp,
+          :true_negatives => tn,
+          :false_negatives => fn,
+          :correct_predictions => 100*(tp+tn).to_f/n,
+          :weighted_area_under_roc => v[:classification_statistics][:weighted_area_under_roc].to_f,
+          :sensitivity => tp.to_f/(tp+fn),
+          :specificity => tn.to_f/(tn+fp),
+        }
+      when "regression"
+        {
+          :nr_predictions => v[:num_instances] - v[:num_unpredicted],
+          :r_square => v[:regression_statistics][:r_square],
+          :root_mean_squared_error => v[:regression_statistics][:root_mean_squared_error],
+          :mean_absolute_error => v[:regression_statistics][:mean_absolute_error],
+        }
+      end
+    end
+
 	end
 end
 

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: opentox-ruby
 version: !ruby/object:Gem::Version 
-  hash: 29
+  hash: 27
   prerelease: false
   segments: 
   - 0
   - 0
-  - 1
-  version: 0.0.1
+  - 2
+  version: 0.0.2
 platform: ruby
 authors: 
 - Christoph Helma, Martin Guetlein, Andreas Maunz, Micha Rautenberg, David Vorgrimmler
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-11-24 00:00:00 +01:00
+date: 2010-11-30 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -201,7 +201,7 @@ dependencies:
   type: :runtime
   version_requirements: *id013
 - !ruby/object:Gem::Dependency 
-  name: tmail
+  name: yajl-ruby
   prerelease: false
   requirement: &id014 !ruby/object:Gem::Requirement 
     none: false
@@ -215,7 +215,7 @@ dependencies:
   type: :runtime
   version_requirements: *id014
 - !ruby/object:Gem::Dependency 
-  name: rinruby
+  name: tmail
   prerelease: false
   requirement: &id015 !ruby/object:Gem::Requirement 
     none: false
@@ -229,7 +229,7 @@ dependencies:
   type: :runtime
   version_requirements: *id015
 - !ruby/object:Gem::Dependency 
-  name: rjb
+  name: rinruby
   prerelease: false
   requirement: &id016 !ruby/object:Gem::Requirement 
     none: false
@@ -243,21 +243,21 @@ dependencies:
   type: :runtime
   version_requirements: *id016
 - !ruby/object:Gem::Dependency 
-  name: dm-core
+  name: rjb
   prerelease: false
   requirement: &id017 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 1
+        hash: 3
         segments: 
-        - 1
-        version: "1"
+        - 0
+        version: "0"
   type: :runtime
   version_requirements: *id017
 - !ruby/object:Gem::Dependency 
-  name: dm-serializer
+  name: dm-core
   prerelease: false
   requirement: &id018 !ruby/object:Gem::Requirement 
     none: false
@@ -271,7 +271,7 @@ dependencies:
   type: :runtime
   version_requirements: *id018
 - !ruby/object:Gem::Dependency 
-  name: dm-timestamps
+  name: dm-serializer
   prerelease: false
   requirement: &id019 !ruby/object:Gem::Requirement 
     none: false
@@ -285,7 +285,7 @@ dependencies:
   type: :runtime
   version_requirements: *id019
 - !ruby/object:Gem::Dependency 
-  name: dm-types
+  name: dm-timestamps
   prerelease: false
   requirement: &id020 !ruby/object:Gem::Requirement 
     none: false
@@ -299,7 +299,7 @@ dependencies:
   type: :runtime
   version_requirements: *id020
 - !ruby/object:Gem::Dependency 
-  name: dm-migrations
+  name: dm-types
   prerelease: false
   requirement: &id021 !ruby/object:Gem::Requirement 
     none: false
@@ -313,7 +313,7 @@ dependencies:
   type: :runtime
   version_requirements: *id021
 - !ruby/object:Gem::Dependency 
-  name: dm-mysql-adapter
+  name: dm-migrations
   prerelease: false
   requirement: &id022 !ruby/object:Gem::Requirement 
     none: false
@@ -327,7 +327,7 @@ dependencies:
   type: :runtime
   version_requirements: *id022
 - !ruby/object:Gem::Dependency 
-  name: dm-validations
+  name: dm-mysql-adapter
   prerelease: false
   requirement: &id023 !ruby/object:Gem::Requirement 
     none: false
@@ -341,9 +341,23 @@ dependencies:
   type: :runtime
   version_requirements: *id023
 - !ruby/object:Gem::Dependency 
-  name: haml
+  name: dm-validations
   prerelease: false
   requirement: &id024 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 1
+        segments: 
+        - 1
+        version: "1"
+  type: :runtime
+  version_requirements: *id024
+- !ruby/object:Gem::Dependency 
+  name: haml
+  prerelease: false
+  requirement: &id025 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -353,11 +367,11 @@ dependencies:
         - 3
         version: "3"
   type: :runtime
-  version_requirements: *id024
+  version_requirements: *id025
 - !ruby/object:Gem::Dependency 
   name: jeweler
   prerelease: false
-  requirement: &id025 !ruby/object:Gem::Requirement 
+  requirement: &id026 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -367,7 +381,7 @@ dependencies:
         - 0
         version: "0"
   type: :development
-  version_requirements: *id025
+  version_requirements: *id026
 description: Ruby wrapper for the OpenTox REST API (http://www.opentox.org)
 email: helma@in-silico.ch
 executables: 
@@ -390,17 +404,22 @@ files:
 - lib/config/config_ru.rb
 - lib/dataset.rb
 - lib/environment.rb
+- lib/feature.rb
 - lib/helper.rb
 - lib/model.rb
+- lib/opentox-ruby.rb
 - lib/opentox.owl
+- lib/opentox.rb
 - lib/overwrite.rb
+- lib/parser.rb
 - lib/rest_client_wrapper.rb
+- lib/serializer.rb
 - lib/spork.rb
 - lib/task.rb
 - lib/templates/config.yaml
 - lib/validation.rb
 has_rdoc: true
-homepage: http://github.com/helma/opentox-ruby-api-wrapper
+homepage: http://github.com/helma/opentox-ruby
 licenses: []
 
 post_install_message: