opentox-ruby 0.0.1

data/lib/overwrite.rb

@@ -0,0 +1,14 @@
+# class overwrites aka monkey patches
+# hack: store sinatra in global var to make url_for and halt methods accessible
+before{ $sinatra = self unless $sinatra }
+
+class Sinatra::Base
+  # overwriting halt to log halts (!= 202)
+  def halt(*response)
+    LOGGER.error "halt "+response.first.to_s+" "+(response.size>1 ? response[1].to_s : "") if response and response.first and response.first >= 300
+    # orig sinatra code:
+    response = response.first if response.length == 1
+    throw :halt, response
+  end
+end
+

data/lib/rest_client_wrapper.rb

@@ -0,0 +1,168 @@
+
+
+module OpenTox
+
+  #PENDING: implement ot error api, move to own file
+  class Error
+    
+    attr_accessor :code, :body, :uri, :payload, :headers
+    
+    def initialize(code, body, uri, payload, headers)
+      self.code = code
+      self.body = body.to_s[0..1000]
+      self.uri = uri
+      self.payload = payload
+      self.headers = headers
+    end
+    
+    def self.parse(error_array_string)
+      begin
+        err = YAML.load(error_array_string)
+        if err and err.is_a?(Array) and err.size>0 and err[0].is_a?(Error)
+          return err
+        else
+          return nil
+        end
+      rescue
+        return nil
+      end
+    end
+    
+  end
+  
+  class WrapperResult < String
+    attr_accessor :content_type, :code
+  end
+  
+  class RestClientWrapper
+    
+    def self.get(uri, headers=nil, wait=true)
+      execute( "get", uri, headers, nil, wait)
+    end
+    
+    def self.post(uri, headers, payload=nil, wait=true)
+      execute( "post", uri, headers, payload, wait )
+    end
+    
+    def self.put(uri, headers, payload=nil )
+      execute( "put", uri, headers, payload )
+    end
+
+    def self.delete(uri, headers=nil)
+      execute( "delete", uri, headers, nil)
+    end
+
+    def self.raise_uri_error(error_msg, uri, headers=nil, payload=nil)
+      do_halt( "-", error_msg, uri, headers, payload )         
+    end
+    
+    private
+    def self.execute( rest_call, uri, headers, payload=nil, wait=true )
+      
+      do_halt 400,"uri is null",uri,headers,payload unless uri
+      do_halt 400,"not a uri",uri,headers,payload unless Utils.is_uri?(uri)
+      do_halt 400,"headers are no hash",uri,headers,payload unless headers==nil or headers.is_a?(Hash)
+      do_halt 400,"nil headers for post not allowed, use {}",uri,headers,payload if rest_call=="post" and headers==nil
+      headers.each{ |k,v| headers.delete(k) if v==nil } if headers #remove keys with empty values, as this can cause problems
+      
+      begin
+        #LOGGER.debug "RestCall: "+rest_call.to_s+" "+uri.to_s+" "+headers.inspect
+        resource = RestClient::Resource.new(uri,{:timeout => 60}) #, :user => @@users[:users].keys[0], :password => @@users[:users].values[0]})
+        if payload
+          result = resource.send(rest_call, payload, headers)
+        elsif headers
+          result = resource.send(rest_call, headers)
+        else
+          result = resource.send(rest_call)
+        end
+        
+        # result is a string, with the additional fields content_type and code
+        res = WrapperResult.new(result.body)
+        res.content_type = result.headers[:content_type]
+        raise "content-type not set" unless res.content_type
+        res.code = result.code
+        
+        return res if res.code==200 || !wait
+        
+        while (res.code==201 || res.code==202)
+          res = wait_for_task(res, uri)
+        end
+        raise "illegal status code: '"+res.code.to_s+"'" unless res.code==200
+        return res
+        
+      rescue RestClient::RequestTimeout => ex
+        do_halt 408,ex.message,uri,headers,payload
+      rescue => ex
+        #raise ex
+        #raise "'"+ex.message+"' uri: "+uri.to_s
+        begin
+          code = ex.http_code
+          msg = ex.http_body
+        rescue
+          code = 500
+          msg = ex.to_s
+        end
+        do_halt code,msg,uri,headers,payload
+      end
+    end
+    
+    def self.wait_for_task( res, base_uri )
+                          
+      task = nil
+      case res.content_type
+      when /application\/rdf\+xml|application\/x-yaml/
+        task = OpenTox::Task.from_data(res, res.content_type, res.code, base_uri)
+      when /text\//
+        raise "uri list has more than one entry, should be a task" if res.content_type=~/text\/uri-list/ and
+          res.split("\n").size > 1 #if uri list contains more then one uri, its not a task
+        task = OpenTox::Task.find(res.to_s) if Utils.task_uri?(res)
+      else
+        raise "unknown content-type for task: '"+res.content_type.to_s+"'" #+"' content: "+res[0..200].to_s
+      end
+      
+      LOGGER.debug "result is a task '"+task.uri.to_s+"', wait for completion"
+      task.wait_for_completion
+      raise task.description unless task.completed? # maybe task was cancelled / error
+      
+      res = WrapperResult.new task.resultURI
+      res.code = task.http_code
+      res.content_type = "text/uri-list"
+      return res
+    end
+    
+    def self.do_halt( code, body, uri, headers, payload=nil )
+      
+      #build error
+      causing_errors = Error.parse(body)
+      if causing_errors
+        error = causing_errors + [Error.new(code, "subsequent error", uri, payload, headers)]
+      else
+        error = [Error.new(code, body, uri, payload, headers)]
+      end
+
+      #debug utility: write error to file       
+      error_dir = "/tmp/ot_errors"
+      FileUtils.mkdir(error_dir) unless File.exist?(error_dir)
+      raise "could not create error dir" unless File.exist?(error_dir) and File.directory?(error_dir)
+      file_name = "error"
+      time=Time.now.strftime("%m.%d.%Y-%H:%M:%S")
+      count = 1
+      count+=1 while File.exist?(File.join(error_dir,file_name+"_"+time+"_"+count.to_s))
+      File.new(File.join(error_dir,file_name+"_"+time+"_"+count.to_s),"w").puts(body)
+      
+      # handle error
+      # we are either in a task, or in sinatra
+      # PENDING: always return yaml for now
+      
+      if $self_task #this global var in Task.as_task to mark that the current process is running in a task
+        raise error.to_yaml # the error is caught, logged, and task state is set to error in Task.as_task
+      #elsif $sinatra  #else halt sinatra
+         #$sinatra.halt(502,error.to_yaml)
+      elsif defined?(halt)         
+         halt(502,error.to_yaml)
+      else #for testing purposes (if classes used directly)
+        raise error.to_yaml
+      end
+    end
+  end
+end

data/lib/spork.rb

@@ -0,0 +1,83 @@
+# A way to cleanly handle process forking in Sinatra when using Passenger, aka "sporking some code".
+# This will allow you to properly execute some code asynchronously, which otherwise does not work correctly.
+#
+# Written by Ron Evans
+# More info at http://deadprogrammersociety.com
+#
+# Mostly lifted from the Spawn plugin for Rails (http://github.com/tra/spawn)
+# but with all of the Rails stuff removed.... cause you are using Sinatra. If you are using Rails, Spawn is
+# what you need. If you are using something else besides Sinatra that is Rack-based under Passenger, and you are having trouble with
+# asynch processing, let me know if spork helped you.
+# 
+module Spork
+  # things to close in child process
+  @@resources = []
+  def self.resources
+    @@resources
+  end
+  
+  # set the resource to disconnect from in the child process (when forking)
+  def self.resource_to_close(resource)
+    @@resources << resource
+  end
+
+  # close all the resources added by calls to resource_to_close
+  def self.close_resources
+    @@resources.each do |resource|
+      resource.close if resource && resource.respond_to?(:close) && !resource.closed?
+    end
+    @@resources = []
+  end
+
+  # actually perform the fork... er, spork
+  # valid options are:
+  # :priority => to set the process priority of the child
+  # :logger => a logger object to use from the child
+  # :no_detach => true if you want to keep the child process under the parent control. usually you do NOT want this
+  def self.spork(options={})
+    logger = options[:logger]
+    logger.debug "spork> parent PID = #{Process.pid}" if logger
+    
+    child = fork do
+      begin
+        start = Time.now
+        logger.debug "spork> child PID = #{Process.pid}" if logger
+
+        # set the nice priority if needed
+        Process.setpriority(Process::PRIO_PROCESS, 0, options[:priority]) if options[:priority]
+
+        # disconnect from the rack
+        Spork.close_resources
+
+        # run the block of code that takes so long
+        yield
+
+      rescue => ex
+        #raise ex
+        logger.error "spork> Exception in child[#{Process.pid}] - #{ex.class}: #{ex.message}" if logger
+      ensure
+        logger.info "spork> child[#{Process.pid}] took #{Time.now - start} sec" if logger
+        # this form of exit doesn't call at_exit handlers
+        exit!(0)
+      end
+    end
+
+    # detach from child process (parent may still wait for detached process if they wish)
+    Process.detach(child) unless options[:no_detach]
+
+    return child
+  end
+  
+end
+
+# Patch to work with passenger
+if defined? Passenger::Rack::RequestHandler
+  class Passenger::Rack::RequestHandler
+    alias_method :orig_process_request, :process_request
+    def process_request(env, input, output)
+      Spork.resource_to_close(input)
+      Spork.resource_to_close(output)
+      orig_process_request(env, input, output)
+    end
+  end
+end

data/lib/task.rb

@@ -0,0 +1,176 @@
+$self_task=nil
+
+module OpenTox
+
+  class Task
+
+    # due_to_time is only set in local tasks 
+    TASK_ATTRIBS = [ :uri, :date, :title, :creator, :description, :hasStatus, :percentageCompleted, :resultURI, :due_to_time ]
+    TASK_ATTRIBS.each{ |a| attr_accessor(a) }
+    attr_accessor :http_code
+    
+    private
+    def initialize(uri)
+      @uri = uri.to_s.strip
+    end
+    
+    # create is private now, use OpenTox::Task.as_task
+    def self.create( params )
+      task_uri = RestClientWrapper.post(@@config[:services]["opentox-task"], params, nil, false).to_s
+      Task.find(task_uri.chomp)
+    end
+  
+    public
+    def self.find( uri, accept_header=nil )
+      task = Task.new(uri)
+      task.reload( accept_header )
+      return task
+    end
+    
+    def self.from_data(data, content_type, code, base_uri)
+      task = Task.new(nil)
+      task.http_code = code
+      task.reload_from_data(data, content_type, base_uri)
+      return task
+    end
+    
+    def reload( accept_header=nil )
+      unless accept_header 
+        if (@@config[:yaml_hosts].include?(URI.parse(uri).host))
+          accept_header = "application/x-yaml"
+        else
+          accept_header = 'application/rdf+xml'
+        end
+      end
+      result = RestClientWrapper.get(uri, {:accept => accept_header}, false)#'application/x-yaml'})
+      @http_code = result.code
+      reload_from_data(result, result.content_type, uri)
+    end
+    
+    def reload_from_data( data, content_type, base_uri )
+      case content_type
+      when /yaml/
+        task =  YAML.load data
+        TASK_ATTRIBS.each do |a|
+          raise "task yaml data invalid, key missing: "+a.to_s unless task.has_key?(a)
+          send("#{a.to_s}=".to_sym,task[a])
+        end
+      when /application\/rdf\+xml/
+        owl = OpenTox::Owl.from_data(data,base_uri,"Task")
+        self.uri = owl.uri
+        (TASK_ATTRIBS-[:uri]).each{|a| self.send("#{a.to_s}=".to_sym, owl.get(a.to_s))}
+      else
+        raise "content type for tasks not supported: "+content_type.to_s
+      end
+      raise "uri is null after loading" unless @uri and @uri.to_s.strip.size>0
+    end
+    
+    def cancel
+      RestClientWrapper.put(File.join(@uri,'Cancelled'))
+      reload
+    end
+
+    def completed(uri)
+      RestClientWrapper.put(File.join(@uri,'Completed'),{:resultURI => uri})
+      reload
+    end
+
+    def error(description)
+      RestClientWrapper.put(File.join(@uri,'Error'),{:description => description.to_s[0..2000]})
+      reload
+    end
+    
+    def pid=(pid)
+      RestClientWrapper.put(File.join(@uri,'pid'), {:pid => pid})
+    end
+
+    def running?
+      @hasStatus.to_s == 'Running'
+    end
+
+    def completed?
+      @hasStatus.to_s == 'Completed'
+    end
+
+    def error?
+      @hasStatus.to_s == 'Error'
+    end
+
+    # waits for a task, unless time exceeds or state is no longer running
+    def wait_for_completion(dur=0.3)
+      
+      if (@uri.match(@@config[:services]["opentox-task"]))
+        due_to_time = (@due_to_time.is_a?(Time) ? @due_to_time : Time.parse(@due_to_time))
+        running_time = due_to_time - (@date.is_a?(Time) ? @date : Time.parse(@date))
+      else
+        # the date of the external task cannot be trusted, offest to local time might be to big
+        due_to_time = Time.new + EXTERNAL_TASK_MAX_DURATION
+        running_time = EXTERNAL_TASK_MAX_DURATION
+      end
+      LOGGER.debug "start waiting for task "+@uri.to_s+" at: "+Time.new.to_s+", waiting at least until "+due_to_time.to_s
+      
+      while self.running?
+        sleep dur
+        reload
+        check_state
+        if (Time.new > due_to_time)
+          raise "max wait time exceeded ("+running_time.to_s+"sec), task: '"+@uri.to_s+"'"
+        end
+      end
+      
+      LOGGER.debug "Task '"+@hasStatus+"': "+@uri.to_s+", Result: "+@resultURI.to_s
+    end
+  
+    def check_state
+      begin
+        raise "illegal task state, task is completed, resultURI is no URI: '"+@resultURI.to_s+
+            "'" unless @resultURI and Utils.is_uri?(@resultURI) if completed?
+        
+        if @http_code == 202
+          raise "illegal task state, code is 202, but hasStatus is not Running: '"+@hasStatus+"'" unless running?
+        elsif @http_code == 201
+          raise "illegal task state, code is 201, but hasStatus is not Completed: '"+@hasStatus+"'" unless completed?
+          raise "illegal task state, code is 201, resultURI is no task-URI: '"+@resultURI.to_s+
+              "'" unless @resultURI and Utils.task_uri?(@resultURI)
+        end
+      rescue => ex
+        RestClientWrapper.raise_uri_error(ex.message, @uri)
+      end
+    end
+  
+    # returns the task uri
+    # catches halts and exceptions, task state is set to error then
+    def self.as_task( title, creator, max_duration=DEFAULT_TASK_MAX_DURATION, description=nil )
+      #return yield nil
+      
+      params = {:title=>title, :creator=>creator, :max_duration=>max_duration, :description=>description }
+      task = OpenTox::Task.create(params)
+      task_pid = Spork.spork(:logger => LOGGER) do
+        LOGGER.debug "Task #{task.uri} started #{Time.now}"
+        $self_task = task
+        
+        begin
+          result = catch(:halt) do
+            yield task
+          end
+          # catching halt, set task state to error
+          if result && result.is_a?(Array) && result.size==2 && result[0]>202
+            LOGGER.error "task was halted: "+result.inspect
+            task.error(result[1])
+            return
+          end
+          LOGGER.debug "Task #{task.uri} done #{Time.now} -> "+result.to_s
+          task.completed(result)
+        rescue => ex
+          LOGGER.error "task failed: "+ex.message
+          LOGGER.error ": "+ex.backtrace.join("\n")
+          task.error(ex.message)
+        end
+      end  
+      task.pid = task_pid
+      LOGGER.debug "Started task: "+task.uri.to_s
+      task.uri
+    end  
+  end
+
+end

data/lib/templates/config.yaml

@@ -0,0 +1,41 @@
+# Example configuration for OpenTox, please adjust to your settings
+# 
+# Database setup:
+#
+# Example MySql:
+#
+:database: 
+  :adapter:  mysql
+  :database: production
+  :username: root
+  :password: opentox
+  :host:     localhost
+#
+# Example 1: Using external test services
+#
+# :services:
+#  opentox-compound:      "http://webservices.in-silico.ch/compound/"
+#  opentox-dataset:       "http://webservices.in-silico.ch/dataset/"
+#  opentox-algorithm:     "http://webservices.in-silico.ch/algorithm/"
+#  opentox-model:         "http://webservices.in-silico.ch/model/"
+#  opentox-task:          "http://webservices.in-silico.ch/task/"
+#  opentox-validation:    "http://opentox.informatik.uni-freiburg.de/validation/"
+#
+# Example 2: Using local services
+:base_dir: /home/ist/webservices
+:webserver: passenger
+:services:
+  opentox-compound:      "http://localhost/compound/"
+  opentox-dataset:       "http://localhost/dataset/"
+  opentox-algorithm:     "http://localhost/algorithm/"
+  opentox-model:         "http://localhost/model/"
+  opentox-task:          "http://localhost/task/"
+  opentox-validation:    "http://localhost/validation/"
+#
+# Yaml capable hosts (faster than OWL-DL)
+#
+:yaml_hosts:
+  - "localhost"
+
+# Uncomment for verbose logging
+# :logger: debug