rbbt-util 5.15.3 → 5.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 63ec9663b0d7998f633493f92b70a30854fe604e
-  data.tar.gz: feaceed53691a6b261e7c0eb17d97e0f70d05a39
+  metadata.gz: 7eb0331120fec20040b7d6d0c2572e5d09b2a423
+  data.tar.gz: 689c858bc997cfc925e2be00a3a71a8c1d708e85
 SHA512:
-  metadata.gz: bf81be699f8921c98657fc085f9295a3895d64d452c8df9bf3f87746ee3ec1487fc3c12ed37bfb9d88b1613b32b8c826cbce5c267e206cf54fd323e10810a24c
-  data.tar.gz: 1161f4eb4c9d9856b258e6d0f0fd2be7aa6d18d9a6b42ade8eff3a143334012bd138d1874fb4e0391a430357a2f0845e96b5ad0a43293df7cfb0dd59bdcdf6a4
+  metadata.gz: 976840c06b1e1c277beaf3852920c1fc6f7be6f25d1eb0b9f41fbbe920e140649ab23bc32e6c081567d7236fe37b6cc3339de4094de407e6a77da023cd591a94
+  data.tar.gz: ad29facb7a20e308c8045ef1ef7134fdd425568408acff53e20db772416b7d7a78989307197427ec163aa96c8aeac83e7f1cb77823bc20d9ea2394ee182d7091

data/lib/rbbt/rest/client.rb

@@ -0,0 +1,56 @@
+require 'rest_client'
+require 'json'
+require 'rbbt/workflow'
+require 'rbbt/workflow/step'
+require 'rbbt/util/misc'
+
+require 'rbbt/rest/client/get'
+require 'rbbt/rest/client/adaptor'
+require 'rbbt/rest/client/step'
+
+class WorkflowRESTClient 
+  include Workflow
+
+  attr_accessor :url, :name, :exec_exports, :asynchronous_exports, :synchronous_exports
+
+  def initialize(url, name)
+    Log.debug{ "Loading remote workflow #{ name }: #{ url }" }
+    @url, @name = url, name
+    init_remote_tasks
+  end
+  
+  def to_s
+    name
+  end
+
+  def job(task, name, inputs)
+    task_info = task_info(task)
+    fixed_inputs = {}
+    input_types = task_info[:input_types]
+
+    inputs.each do |k,v| 
+      k = k.to_sym
+      if TSV === v
+        fixed_inputs[k] = v.to_s
+      else
+        case input_types[k].to_sym
+        when :tsv, :array, :file, :text
+          fixed_inputs[k] = (String === v and Open.exists?(v)) ? Open.open(v) : v
+        else
+          fixed_inputs[k] = v
+        end
+      end
+    end
+
+    RemoteStep.new(url, task, name, fixed_inputs, task_info[:result_type], task_info[:result_description], @exec_exports.include?(task))
+  end
+
+  def load_id(id)
+    task, name = id.split("/")
+    step = RemoteStep.new url, task, nil
+    step.name = name
+    step.result_type = task_info(task)[:result_type]
+    step.result_description = task_info(task)[:result_description]
+    step
+  end
+end

data/lib/rbbt/rest/client/adaptor.rb

@@ -0,0 +1,66 @@
+class WorkflowRESTClient
+
+  def workflow_description
+    WorkflowRESTClient.get_raw(File.join(url, 'description'))
+  end
+
+  def documentation
+    @documention ||= IndiferentHash.setup(WorkflowRESTClient.get_json(File.join(url, "documentation"),{}))
+  end
+
+  def task_info(task)
+    @task_info ||= {}
+    @task_info[task]
+    
+    if @task_info[task].nil?
+      task_info = WorkflowRESTClient.get_json(File.join(url, task.to_s, 'info'))
+      task_info = WorkflowRESTClient.fix_hash(task_info)
+
+      task_info[:result_type] = task_info[:result_type].to_sym
+      task_info[:export] = task_info[:export].to_sym
+      task_info[:input_types] = WorkflowRESTClient.fix_hash(task_info[:input_types], true)
+      task_info[:inputs] = task_info[:inputs].collect{|input| input.to_sym }
+
+      @task_info[task] = task_info
+    end
+    @task_info[task]
+  end
+
+  def exported_tasks
+    (@asynchronous_exports  + @synchronous_exports + @exec_exports).compact.flatten
+  end
+
+  def tasks
+    @tasks ||= Hash.new do |hash,task_name| 
+      info = task_info(task_name)
+      task = Task.setup info do |*args|
+        raise "This is a remote task" 
+      end
+      task.name = task_name.to_sym
+      hash[task_name] = task
+    end
+  end
+
+  def load_tasks
+    exported_tasks.each{|name| tasks[name]}
+    nil
+  end
+
+  def task_dependencies
+    @task_dependencies ||= Hash.new do |hash,task| 
+      hash[task] = if exported_tasks.include? task
+        WorkflowRESTClient.get_json(File.join(url, task.to_s, 'dependencies'))
+      else
+        []
+      end
+    end
+  end
+
+  def init_remote_tasks
+    task_exports = WorkflowRESTClient.get_json(url)
+    @asynchronous_exports = task_exports["asynchronous"].collect{|task| task.to_sym }
+    @synchronous_exports = task_exports["synchronous"].collect{|task| task.to_sym }
+    @exec_exports = task_exports["exec"].collect{|task| task.to_sym }
+    nil
+  end
+end

data/lib/rbbt/rest/client/get.rb

@@ -0,0 +1,96 @@
+class WorkflowRESTClient
+  def self.fix_hash(hash, fix_values = false)
+    fixed = {}
+    hash.each do |key, value|
+      fixed[key.to_sym] = case value
+                          when TrueClass
+                            value
+                          when FalseClass
+                            value
+                          when Hash 
+                            fix_hash(value)  
+                          when (fix_values and String )
+                            value.to_sym
+                          when IO
+                            value.read
+                          when TSV::Dumper
+                            value.stream
+                          when Step
+                            stream = get_stream(value)
+                            stream || value.load
+                          else
+                            value
+                          end
+    end
+    fixed
+  end
+
+  def self.capture_exception
+    begin
+      yield
+    rescue Exception => e
+      raise e unless e.respond_to? :response
+      begin
+        klass, message = e.response.to_s.split " => "
+        klass = Kernel.const_get klass
+        raise klass.new message
+      rescue
+        raise e
+      end
+      raise $!
+    end
+  end
+
+  def self.get_raw(url, params = {})
+    Log.debug{ "RestClient get_raw: #{ url } - #{Misc.fingerprint params}" }
+    params = params.merge({ :_format => 'raw' })
+    capture_exception do
+      Misc.insist(2, 0.5) do
+        RestClient.get(URI.encode(url), :params => params)
+      end
+    end
+  end
+ 
+  def self.post_jobname(url, params = {})
+    Log.debug{ "RestClient post_jobname: #{ url } - #{Misc.fingerprint params}" }
+    params = params.merge({ :_format => 'jobname' })
+
+    capture_exception do
+      RestClient.post(URI.encode(url), params)
+    end
+  end
+  
+  def self.get_json(url, params = {})
+    Log.debug{ "RestClient get_json: #{ url } - #{Misc.fingerprint params }" }
+    params = params.merge({ :_format => 'json' })
+
+    res = capture_exception do
+      RestClient.get(URI.encode(url), :params => params)
+    end
+
+    begin
+      JSON.parse(res)
+    rescue
+      res
+    end
+  end
+
+  def self.post_json(url, params = {})
+    if url =~ /_cache_type=:exec/
+      JSON.parse(Open.open(url, :nocache => true))
+    else
+      params = params.merge({ :_format => 'json' })
+
+      res = capture_exception do
+        RestClient.post(URI.encode(url), params)
+      end
+
+      begin
+        JSON.parse(res)
+      rescue
+        res
+      end
+    end
+  end
+
+end

data/lib/rbbt/rest/client/step.rb

@@ -0,0 +1,164 @@
+class WorkflowRESTClient
+  class RemoteStep < Step
+
+    attr_accessor :url, :base_url, :task, :base_name, :inputs, :result_type, :result_description, :is_exec
+
+    def self.get_streams(inputs)
+      inputs.each do |k,v|
+        if Step === v
+          stream = v.get_stream
+          inputs[k] = stream || v.load
+        end
+      end
+    end
+    def initialize(base_url, task = nil, base_name = nil, inputs = nil, result_type = nil, result_description = nil, is_exec = false)
+      @base_url, @task, @base_name, @inputs, @result_type, @result_description, @is_exec = base_url, task, base_name, inputs, result_type, result_description, is_exec
+      @mutex = Mutex.new
+      RemoteStep.get_streams @inputs
+    end
+
+    def name
+      return nil if @is_exec
+      (Array === @url ? @url.first : @url).split("/").last
+    end
+
+    def task_name
+      (Array === @url ? @url.first : @url).split("/")[-2]
+    end
+
+    def info(check_lock=false)
+      @info ||= begin
+                  init_job unless url
+                  info = WorkflowRESTClient.get_json(File.join(url, 'info'))
+                  info = WorkflowRESTClient.fix_hash(info)
+                  info[:status] = info[:status].to_sym if String === info[:status]
+                  info
+                end
+    end
+    
+    def status
+      begin
+        info[:status]
+      ensure
+        @info = nil
+      end
+    end
+
+    def done?
+      @done || status.to_s == 'done'
+    end
+
+    def files
+      WorkflowRESTClient.get_json(File.join(url, 'files'))
+    end
+
+    def file(file)
+      WorkflowRESTClient.get_raw(File.join(url, 'file', file))
+    end
+
+    #{{{ MANAGEMENT
+    
+    def init_job(cache_type = :asynchronous)
+      @name ||= Persist.memory("RemoteSteps", :jobname => @name, :inputs => inputs) do
+        WorkflowRESTClient.post_jobname(File.join(base_url, task.to_s), inputs.merge(:jobname => @name||@base_name, :_cache_type => cache_type))
+      end
+      @url = File.join(base_url, task.to_s, @name)
+      nil
+    end
+
+    def load_res(res, result_type = nil)
+      result_type ||= self.result_type
+      case result_type
+      when :string
+        res
+      when :boolean
+        res == "true"
+      when :tsv
+        TSV.open(StringIO.new(res))
+      when :annotations
+        Annotated.load_tsv(TSV.open(StringIO.new(res)))
+      when :array
+        res.split("\n")
+      else
+        JSON.parse res
+      end
+    end
+
+    def get
+      params ||= {}
+      params = params.merge(:_format => [:string, :boolean, :tsv, :annotations,:array].include?(result_type.to_sym) ? :raw : :json )
+      Misc.insist 3, rand(2) + 1 do
+        begin
+          WorkflowRESTClient.get_raw(url, params)
+        rescue
+          Log.exception $!
+          raise $!
+        end
+      end
+    end
+
+    def load
+      params = {}
+      load_res get
+    end
+    
+    def exec_job
+      res = WorkflowRESTClient.capture_exception do
+        RestClient.post(URI.encode(File.join(base_url, task.to_s)), inputs.merge(:_cache_type => :exec, :_format => [:string, :boolean, :tsv, :annotations].include?(result_type) ? :raw : :json))
+      end
+      load_res res, result_type == :array ? :json : result_type
+    end
+
+    def fork
+      init_job(:asynchronous)
+    end
+
+    def running?
+      ! %w(done error aborted).include? status.to_s
+    end
+
+    def path
+      url
+    end
+
+    def run(noload = false)
+      @mutex.synchronize do
+        @result ||= begin
+                      if @is_exec
+                        exec_job 
+                      else
+                        init_job(:synchronous) 
+                        self.load
+                      end
+                    end
+      end
+      noload ? path : @result
+    end
+
+    def exec(*args)
+      exec_job
+    end
+
+    def join
+      return if self.done?
+      self.load
+      self
+    end
+
+    def recursive_clean
+      begin
+        inputs[:_update] = :recursive_clean
+      rescue Exception
+      end
+      self
+    end
+
+    def clean
+      begin
+        inputs[:_update] = :clean
+      rescue Exception
+      end
+      self
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.15.3
+  version: 5.16.0
 platform: ruby
 authors:
 - Miguel Vazquez
@@ -174,6 +174,10 @@ files:
 - lib/rbbt/resource/rake.rb
 - lib/rbbt/resource/util.rb
 - lib/rbbt/resource/with_key.rb
+- lib/rbbt/rest/client.rb
+- lib/rbbt/rest/client/adaptor.rb
+- lib/rbbt/rest/client/get.rb
+- lib/rbbt/rest/client/step.rb
 - lib/rbbt/tsv.rb
 - lib/rbbt/tsv/accessor.rb
 - lib/rbbt/tsv/attach.rb