rbbt-util 5.26.157 → 5.26.158

data/lib/rbbt/workflow/remote_workflow/remote_step/rest.rb

@@ -0,0 +1,148 @@
+class RemoteStep
+  module REST
+
+    def get
+      params ||= {}
+      params = params.merge(:_format => [:string, :boolean, :tsv, :annotations, :array].include?(result_type.to_sym) ? :raw : :json )
+      @cache_result ||= Persist.persist("REST persist", :binary, :file => cache_file + "." + Misc.obj2digest(params)) do
+        Misc.insist 3, rand(2) + 1 do
+          begin
+            init_job if url.nil?
+            @adaptor.get_raw(url, params)
+          rescue
+            Log.exception $!
+            raise $!
+          end
+        end
+      end
+    end
+
+    def load
+      params = {}
+      join unless done? or streaming?
+      raise get_exception if error? or aborted?
+      load_res get
+    end
+
+    def exec_job
+      res = _run_job(:exec)
+      load_res res, result_type == :array ? :json : result_type
+    end
+
+    def abort
+      return self if status == :done
+      @adaptor.get_json(@url + '?_update=abort') if @url and @name
+      self
+    end
+
+    def init_job(cache_type = nil, other_params = {})
+      cache_type = :asynchronous if cache_type.nil? and not @is_exec
+      cache_type = :exec if cache_type.nil?
+      @last_info_time = nil
+      @done = false
+      get_streams
+      @name ||= Persist.memory("RemoteSteps", :workflow => self, :task => task, :jobname => @name, :inputs => inputs, :cache_type => cache_type) do
+        Misc.insist do
+          @adaptor.post_jobname(File.join(base_url, task.to_s), inputs.merge(other_params).merge(:jobname => @name||@base_name, :_cache_type => cache_type))
+        end
+      end
+      if Open.remote? @name
+        @url = @name
+        @name = File.basename(@name)
+      else
+        @url = File.join(base_url, task.to_s, @name)
+      end
+      self
+    end
+
+    def recursive_clean
+      Log.warn "Not doing recursive cleans"
+      return
+      begin
+        _restart
+        params = {:_update => :recursive_clean}
+        @adaptor.get_raw(url, params)
+      rescue Exception
+        Log.exception $!
+      end
+      self
+    end
+
+    def _clean
+      begin
+        _restart
+        cache_files.each do |cache_file|
+          Open.rm cache_file
+        end
+        params = {:_update => :clean}
+        @adaptor.clean_url(url, params) if @url
+      rescue Exception
+        Log.exception $!
+      end
+    end
+
+    def clean
+      init_job
+      _clean
+      self
+    end
+
+    def stream_job(task_url, task_params, stream_input, cache_type = :exec)
+      require 'rbbt/util/misc/multipart_payload'
+      WorkflowRESTClient.capture_exception do
+        @streaming = true
+
+        Log.debug{ "RestClient stream #{Process.pid}: #{ task_url } #{stream_input} #{cache_type} - #{Misc.fingerprint task_params}" }
+        res = RbbtMutiplartPayload.issue task_url, task_params, stream_input, nil, nil, true
+        type = res.gets
+
+        out = case type.strip
+              when "LOCATION"
+                @url = res.gets
+                @url.sub!(/\?.*/,'')
+                join
+                WorkflowRESTClient.get_raw(@url)
+                @done = true
+                @streaming = false
+              when /STREAM: (.*)/
+                @url = $1.strip
+                res.callback = Proc.new do
+                  Log.medium "Done streaming result from #{@url}"
+                  @done = true
+                  @streaming = false
+                end
+                res
+              when "BULK"
+                begin
+                  res.read
+                ensure
+                  @done = true
+                  @streaming = false
+                end
+              else
+                raise "What? " + type
+              end
+
+        ConcurrentStream.setup(out, :filename => @url)
+
+        out
+      end
+    end
+
+    def _run_job(cache_type = :async)
+      get_streams
+
+      task_url = URI.encode(File.join(base_url, task.to_s))
+      @adaptor.__prepare_inputs_for_restclient(inputs)
+      task_params = inputs.merge(:_cache_type => cache_type, :jobname => base_name, :_format => [:string, :boolean, :tsv, :annotations].include?(result_type) ? :raw : :json)
+
+      if cache_type == :stream or cache_type == :exec and stream_input and inputs[stream_input]
+        io =  self.stream_job(task_url, task_params, stream_input, cache_type) 
+        return io
+      else
+        @adaptor.execute_job(base_url, task, task_params, cache_type)
+      end
+
+    end
+  end
+end

data/lib/rbbt/workflow/remote_workflow/remote_step/ssh.rb

@@ -0,0 +1,73 @@
+class RemoteStep
+  module SSH
+    attr_accessor :override_dependencies
+
+    def init_job(cache_type = nil, other_params = {})
+      cache_type = :asynchronous if cache_type.nil? and not @is_exec
+      cache_type = :exec if cache_type.nil?
+      @last_info_time = nil
+      @done = false
+      @server, @server_path = RemoteWorkflow::SSH.parse_url base_url
+      @input_id ||= "inputs-" << rand(100000).to_s
+
+      if override_dependencies
+
+        if override_dependencies && override_dependencies.any?
+          override_dependencies.each do |od|
+            name, _sep, value = od.partition("=")
+            inputs[name] = value
+          end
+        end
+
+        RemoteWorkflow::SSH.upload_inputs(@server, inputs, @input_types, @input_id)
+      else
+        RemoteWorkflow::SSH.upload_inputs(@server, inputs, @input_types, @input_id)
+      end
+
+      @name ||= Persist.memory("RemoteSteps", :workflow => self, :task => task, :jobname => @name, :inputs => inputs, :cache_type => cache_type) do
+        Misc.insist do
+          input_types = {}
+          RemoteWorkflow::SSH.post_job(File.join(base_url, task.to_s), @input_id, @base_name)
+        end
+      end
+      if Open.remote? @name
+        @url = @name
+        @name = File.basename(@name)
+      else
+        @url = File.join(base_url, task.to_s, @name)
+      end
+      self
+    end
+
+    def path
+      @server, @server_path = RemoteWorkflow::SSH.parse_url @base_url
+      "ssh://" + @server + ":" + @remote_path
+    end
+
+    def produce(*args)
+      input_types = {}
+      init_job
+      @remote_path = RemoteWorkflow::SSH.run_job(File.join(base_url, task.to_s), @input_id, @base_name)
+      while ! done?
+        sleep 1
+      end
+    end
+
+    def load
+      load_res Open.open(path)
+    end
+
+    def run(*args)
+      produce(*args)
+      self.load unless args.first
+    end
+
+    def clean
+      init_job
+      RemoteStep::SSH.clean(@url, @input_id, @base_name) if done?
+      _restart
+    end
+
+  end
+end
+

data/lib/rbbt/workflow/remote_workflow/remote_step.rb

@@ -0,0 +1,329 @@
+require 'rbbt/workflow'
+
+class RemoteStep < Step
+
+  attr_accessor :url, :base_url, :task, :base_name, :inputs, :input_types, :result_type, :result_description, :is_exec, :is_stream, :stream_input
+
+  def initialize(base_url, task = nil, base_name = nil, inputs = nil, input_types = nil, result_type = nil, result_description = nil, is_exec = false, is_stream = false, stream_input = nil)
+    @base_url, @task, @base_name, @inputs, @input_types, @result_type, @result_description, @is_exec, @is_stream, @stream_input = base_url, task, base_name, inputs, input_types, result_type, result_description, is_exec, is_stream, stream_input
+    @base_url = "http://" << @base_url unless @base_url =~ /^[a-z]+:\/\//
+    @mutex = Mutex.new
+    rest = base_url.include?('ssh:') ? false : true
+
+    if rest
+      @adaptor = RemoteWorkflow::REST
+      self.extend RemoteStep::REST
+    else
+      @adaptor = RemoteWorkflow::SSH
+      self.extend RemoteStep::SSH
+    end
+
+  end
+
+  def clean_name
+    @base_name
+  end
+
+  def cache_file
+    digest = Misc.obj2digest([base_url, task, base_name, inputs])
+    Rbbt.var.cache.REST[[task, clean_name, digest] * "."].find
+  end
+
+  def cache_files
+    Dir.glob(cache_file + '.*')
+  end
+
+  def run(no_load = false)
+    no_load = @is_stream ? :stream : true if no_load
+
+    @result ||= @mutex.synchronize do
+      begin
+        if @is_exec
+          exec(no_load)
+        elsif no_load == :stream
+          _run_job(:stream)
+        elsif no_load
+          init_job 
+          nil
+        else
+          if ! done?
+            init_job 
+            join
+          end
+          self.load
+        end
+      ensure
+        @started = true
+      end
+    end
+
+    return @result if no_load == :stream
+    no_load ? Misc.add_GET_param(path, "_format", "raw") : @result
+  end
+
+
+  def self.get_streams(inputs, stream_input = nil)
+    new_inputs = {}
+    inputs.each do |k,v|
+      stream = stream_input.to_s == k.to_s
+      if Step === v 
+        unless (v.done? or v.streaming?) # or RestClient::Step === v)
+          v.run(true) and v.grace 
+        end
+
+        begin
+          if stream
+            new_inputs[k] = TSV.get_stream(v)
+          else
+            new_inputs[k] = v.load
+          end
+        rescue Exception
+          raise $!
+        end
+      else
+        new_inputs[k] = v
+      end
+    end
+    new_inputs
+  end
+
+  def get_streams
+    return if @inputs_done
+    @inputs = RemoteStep.get_streams @inputs, @stream_input
+    @inputs_done = true
+    @inputs
+  end
+
+  def dup_inputs
+    return if @dupped or ENV["RBBT_NO_STREAM"] == 'true'
+    Log.low "Dupping inputs for #{path}"
+    dupped_inputs = {}
+    @inputs.collect do |k,input|
+      dupped_inputs[k] = Step.dup_stream input
+    end
+    @inputs = dupped_inputs
+    @dupped = true
+  end
+
+  def name
+    return nil if @is_exec
+    return @path if @url.nil?
+    (Array === @url ? @url.first : @url).split("/").last
+  end
+
+  def name=(name)
+    @url = [base_url,task, name] * "/"
+  end
+
+  def task_name
+    return task if task
+    init_job
+    (Array === @url ? @url.first : @url).split("/")[-2]
+  end
+
+  def nopid?
+    false
+  end
+
+  def info(check_lock=false)
+    @done = @info && @info[:status] && (@info[:status].to_sym == :done || @info[:status].to_sym == :error)
+
+    if !@done && (@last_info_time.nil? || (Time.now - @last_info_time) > 0.5)
+      update = true 
+    else
+      update = false
+    end
+
+    @info = Persist.memory("RemoteSteps Info", :url => @url, :persist => true, :update => update) do
+      @last_info_time = Time.now
+      init_job unless @url
+      info = @adaptor.get_json(File.join(@url, 'info'))
+      info = RemoteWorkflow.fix_hash(info)
+      info[:status] = info[:status].to_sym if String === info[:status]
+      info
+    end
+    @info
+  end
+
+  def status
+    return :done if @done
+    return nil unless url or started?
+    #return :streaming if @streaming 
+    begin
+      status = info[:status]
+      @done = true if status and status.to_sym == :done
+      status
+    rescue
+      Log.exception $!
+      nil
+    ensure
+      @info = nil
+    end
+  end
+
+  def started?
+    @result != nil || @started || @streaming
+  end
+
+  def done?
+    return true if cache_files.any?
+    self.init_job unless @url
+    @done || status.to_s == 'done' || status.to_s == 'noinfo'
+  end
+
+  def files
+    @adaptor.get_json(File.join(url, 'files'))
+  end
+
+  def file(file)
+    @adaptor.get_raw(File.join(url, 'file', file))
+  end
+
+  def get_stream
+    case @result
+    when IO 
+      @result
+    when String
+      StringIO.new @result
+    else
+      nil
+    end
+  end
+
+  def grace
+    produce unless @started
+    sleep 0.1 unless started?
+    sleep 0.5 unless started?
+    sleep 1 unless started?
+    while not (done? or started?)
+      sleep 1 
+    end
+  end
+
+  #{{{ MANAGEMENT
+
+
+  def path
+    if @url
+      Misc.add_GET_param(@url, "_format", "raw")
+    elsif @base_name
+      [base_url, task, @base_name + '-' +  Misc.fingerprint(inputs)] * "/"
+    else
+      nil
+    end
+  end
+
+  def fork(noload=false, semaphore=nil)
+    init_job(:asynchronous)
+  end
+
+  def running?
+    ! %w(done error aborted noinfo).include? status.to_s
+  end
+
+  def exec(noload = false)
+    @result ||= begin
+                  if noload == :stream
+                    _run_job(:exec)
+                  else
+                    exec_job 
+                  end
+                ensure
+                  @started = true
+                end
+  end
+
+  def join
+    return true if cache_files.any?
+    init_job unless @url
+    Log.debug{ "Joining RemoteStep: #{path}" }
+    if IO === @result
+      res = @result
+      @result = nil
+      Misc.consume_stream(res, true) 
+    end
+
+    if not (self.done? || self.aborted? || self.error?)
+      self.info 
+      return self if self.done? || self.aborted? || self.error?
+      sleep 0.2 unless self.done? || self.aborted? || self.error?
+      sleep 1 unless self.done? || self.aborted? || self.error?
+      while not (self.done? || self.aborted? || self.error?)
+        sleep 3
+      end
+    end
+
+    self
+  end
+
+  def load_res(res, result_type = nil)
+
+    stream = true if res.respond_to? :read
+    join unless stream
+    result_type ||= self.result_type
+
+    case result_type.to_sym
+    when :string
+      stream ? res.read : res
+    when :boolean
+      (stream ? res.read : res) == 'true'
+    when :tsv
+      if stream
+        TSV.open(res, :monitor => true)
+      else
+        TSV.open(StringIO.new(res))
+      end
+    when :annotations
+      if stream
+        Annotated.load_tsv(TSV.open(res))
+      else
+        Annotated.load_tsv(TSV.open(StringIO.new(res)))
+      end
+    when :array
+      (stream ? res.read : res).split("\n")
+    else
+      json_text = if IO === res
+                    res.read
+                  else
+                    res
+                  end
+      begin
+        JSON.parse json_text
+      rescue
+        case
+        when json_text =~ /^\d+$/
+          json_text.to_i
+        when json_text =~ /^\d+\.\d/
+          json_text.to_f
+        else
+          raise $!
+        end
+      end
+    end
+  end
+
+  def _restart
+    @done = nil
+    @name = nil
+    @started = nil
+    @aborted = nil
+    new_inputs = {}
+    inputs.each do |k,i| 
+      if File === i 
+        new_inputs[k] = File.open(i.path)
+      else
+        new_inputs[k] = i
+      end
+    end
+    @inputs = new_inputs
+    @info = nil
+  end
+
+  def input_checks
+    []
+  end
+
+end
+
+require 'rbbt/workflow/remote_workflow/remote_step/rest'
+require 'rbbt/workflow/remote_workflow/remote_step/ssh'

data/lib/rbbt/workflow/{remote/client.rb → remote_workflow.rb}

@@ -1,17 +1,5 @@
-require 'json'
 require 'rbbt/workflow'
-require 'rbbt/workflow/step'
-require 'rbbt/util/misc'
-
-require 'rbbt/workflow/remote/remote_step'
-
-require 'rbbt/workflow/remote/rest/get'
-require 'rbbt/workflow/remote/rest/adaptor'
-
-require 'rbbt/workflow/remote/ssh/get'
-require 'rbbt/workflow/remote/ssh/adaptor'
-
-class WorkflowRemoteClient 
+class RemoteWorkflow
   include Workflow
 
   attr_accessor :url, :name, :exec_exports, :synchronous_exports, :asynchronous_exports, :stream_exports
@@ -23,10 +11,11 @@ class WorkflowRemoteClient
     rest = url.include?('ssh://') ? false : true
 
     if rest
-      self.extend WorkflowRESTClient
+      self.extend RemoteWorkflow::REST
     else
-      self.extend WorkflowSSHClient
+      self.extend RemoteWorkflow::SSH
     end
+
     init_remote_tasks
   end
 
@@ -55,7 +44,7 @@ class WorkflowRemoteClient
     end
 
     stream_input = @can_stream ? task_info(task)[:input_options].select{|k,o| o[:stream] }.collect{|k,o| k }.first : nil
-    RemoteStep.new(url, task, name, fixed_inputs, task_info[:result_type], task_info[:result_description], @exec_exports.include?(task), @stream_exports.include?(task), stream_input)
+    RemoteStep.new(url, task, name, fixed_inputs, task_info[:input_types], task_info[:result_type], task_info[:result_description], @exec_exports.include?(task), @stream_exports.include?(task), stream_input)
   end
 
   def load_id(id)
@@ -67,3 +56,7 @@ class WorkflowRemoteClient
     step
   end
 end
+
+require 'rbbt/workflow/remote_workflow/driver'
+require 'rbbt/workflow/remote_workflow/remote_step'
+

data/lib/rbbt/workflow/util/archive.rb

@@ -1,4 +1,3 @@
-require 'rbbt/workflow/remote/ssh/driver'
 class Step
 
   MAIN_RSYNC_ARGS="-avztAXHP"
@@ -132,7 +131,7 @@ class Step
     recursive = false if recursive.nil?
 
     paths = if options[:source]
-              SSHDriver.run(options[:source], <<-EOF).split("\n")
+              Misc.ssh_run(options[:source], <<-EOF).split("\n")
 require 'rbbt-util'
 require 'rbbt/workflow'
 
@@ -165,7 +164,7 @@ puts files * "\n"
 
 
     target = if options[:target] 
-               target = SSHDriver.run(options[:target], <<-EOF).split("\n").first
+               target = Misc.ssh_run(options[:target], <<-EOF).split("\n").first
 require 'rbbt-util'
 path = "var/jobs"
 resource = #{resource.to_s}

data/lib/rbbt/workflow.rb

@@ -48,13 +48,13 @@ module Workflow
   end
 
   def self.require_remote_workflow(wf_name, url)
-    require 'rbbt/workflow/remote/client'
-    eval "Object::#{wf_name} = WorkflowRemoteClient.new '#{ url }', '#{wf_name}'"
+    require 'rbbt/workflow/remote_workflow'
+    eval "Object::#{wf_name} = RemoteWorkflow.new '#{ url }', '#{wf_name}'"
   end
 
   def self.require_remote_workflow(wf_name, url)
-    require 'rbbt/workflow/remote/client'
-    eval "Object::#{wf_name} = WorkflowRemoteClient.new '#{ url }', '#{wf_name}'"
+    require 'rbbt/workflow/remote_workflow'
+    eval "Object::#{wf_name} = RemoteWorkflow.new '#{ url }', '#{wf_name}'"
   end
 
   def self.load_workflow_libdir(filename)

data/share/rbbt_commands/workflow/remote/add

@@ -1,9 +1,35 @@
 #!/usr/bin/env ruby
 
-require 'rbbt'
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+
+$0 = "rbbt #{$previous_commands*" "} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Description
+
+$ #{$0} [options] <workflow> <url>
+
+Use - to read from STDIN
+
+-h--help Print this help
+
+EOF
+if options[:help]
+  if defined? rbbt_usage
+    rbbt_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
 
 workflow, url = ARGV
-url = File.join(url, workflow) unless url =~ /\/#{workflow}$/
+
+raise ParameterException unless workflow && url
+
+url = File.join(url, workflow) unless url =~ /[\/\:]#{workflow}\/?$/
 config_file = Rbbt.etc.remote_workflows
 remote_workflows = config_file.exists? ? config_file.yaml : {}
 remote_workflows[workflow] = url

data/share/rbbt_commands/workflow/task

@@ -235,6 +235,7 @@ end
 Workflow.workdir = Path.setup(File.expand_path(options.delete(:workdir_all))) if options[:workdir_all]
 
 workflow = Workflow.require_workflow workflow
+workflow.init_remote_tasks
 
 if clean_task
   ENV["RBBT_UPDATE"] = 'true'
@@ -363,7 +364,7 @@ begin
 
     result_type = job.result_type
 
-    res = JSON.parse(res.read) if (defined?(WorkflowRemoteClient) and WorkflowRemoteClient::RemoteStep === job) && %w(array float integer boolean).include?(result_type.to_s)
+    res = JSON.parse(res.read) if (defined?(RemoteStep) and RemoteStep === job) && %w(array float integer boolean).include?(result_type.to_s)
 
     case
     when res.respond_to?(:gets)
@@ -469,7 +470,7 @@ begin
       Log.clear_line 
     end
 
-    if Open.remote? job.path
+    if Open.remote?(job.path)
       out.puts job.path + Log.color(:blue, "?_format=raw")
     else
       out.puts job.path
@@ -550,7 +551,7 @@ when Step
     end
   else
     res.join
-    out.puts Open.read(res.path) if File.exist? res.path
+    out.puts Open.read(res.path) if Open.exist?(res.path) || Open.remote?(res.path) || Open.ssh?(res.path)
   end
 else
   if Array === res