rbbt-util 5.44.1 → 6.0.3

data/lib/rbbt/workflow/refactor.rb

@@ -0,0 +1,153 @@
+require_relative 'refactor/export'
+require_relative 'refactor/recursive'
+require_relative 'refactor/task_info'
+require_relative 'refactor/inputs'
+require_relative 'refactor/entity'
+
+class Step
+  alias get_stream stream
+  alias old_exec exec
+
+  def exec(noload = false)
+    old_exec
+  end
+
+  def self.md5_file(path)
+    path.nil? ? nil : path + '.md5'
+  end
+
+  def md5_file
+    Step.md5_file(path)
+  end
+
+  alias real_inputs non_default_inputs
+
+  def reset_info(info = {})
+    if ENV["BATCH_SYSTEM"]
+      info = info.dup
+      info[:batch_system] = ENV["BATCH_SYSTEM"]
+      info[:batch_job] = ENV["BATCH_JOB_ID"]
+    end
+    save_info(info)
+  end
+end
+
+module Workflow
+  alias workdir= directory=
+  
+  def resumable
+    Log.warn "RESUMABLE MOCKED"
+  end
+
+  DEFAULT_NAME = Task::DEFAULT_NAME
+
+end
+
+module ComputeDependency
+  attr_accessor :compute
+  def self.setup(dep, value)
+    dep.extend ComputeDependency
+    dep.compute = value
+  end
+
+  def canfail?
+    compute == :canfail || (Array === compute && compute.include?(:canfail))
+  end
+end
+
+class Step
+
+  def soft_grace
+    sleep 1 until Open.exists?(info_file)
+  end
+end
+
+Rbbt.relay_module_method Workflow, :load_step, Step, :load
+Rbbt.relay_module_method Workflow, :fast_load_step, Step, :load
+
+module Workflow
+  attr_accessor :remote_tasks
+  def remote_tasks
+    @remote_tasks ||= {}
+  end
+  def task_for(path)
+    parts = path.split("/")
+    if parts.include?(self.to_s)
+      parts[parts.index(self.to_s) + 1]
+    else
+      parts[-2]
+    end
+  end
+
+  def fast_load_id(id)
+    path = if Path === directory
+             directory[id].find
+           else
+             File.join(directory, id)
+           end
+    task = task_for path
+    return remote_tasks[task].load_id(id) if remote_tasks && remote_tasks.include?(task)
+    return Workflow.fast_load_step path
+  end
+
+  alias load_id fast_load_id
+
+  class << self
+    alias original_require_workflow require_workflow
+  
+    def require_remote_workflow(wf_name, url)
+      require 'rbbt/workflow/remote_workflow'
+      eval "Object::#{wf_name.split("+").first} = RemoteWorkflow.new '#{ url }', '#{wf_name}'"
+    end
+
+    def require_workflow(wf_name, force_local = true)
+      if Open.remote?(wf_name) or Open.ssh?(wf_name)
+        url = wf_name
+
+        if Open.ssh?(wf_name)
+          wf_name = File.basename(url.split(":").last)
+        else
+          wf_name = File.basename(url)
+        end
+
+        begin
+          return require_remote_workflow(wf_name, url)
+        ensure
+          Log.debug{"Workflow #{ wf_name } loaded remotely: #{ url }"}
+        end
+      end
+
+      original_require_workflow(wf_name)
+    end
+  end
+end
+
+module Workflow
+  class << self
+    def workflow_dir
+      @workflow_dir || 
+        ENV["RBBT_WORKFLOW_DIR"] || 
+        begin 
+          workflow_dir_config = Path.setup("etc/workflow_dir")
+          if workflow_dir_config.exists?
+            Path.setup(workflow_dir_config.read.strip)
+          else
+            Path.setup('workflows').find(:user)
+          end
+        end
+    end
+
+    def workflow_repo
+      @workflow_repo || 
+        ENV["RBBT_WORKFLOW_REPO"] || 
+        begin 
+          workflow_repo_config = Path.setup("etc/workflow_repo")
+          if workflow_repo_config.exists?
+            workflow_repo_config.read.strip
+          else
+            'https://github.com/Rbbt-Workflows/'
+          end
+        end
+    end
+  end
+end

data/lib/rbbt/workflow/remote_workflow/driver/ssh.rb

@@ -80,7 +80,7 @@ res = job_info = job.info
       script =<<-EOF
 jobname = #{jobname.nil? ? 'nil' : "'#{jobname}'"}
 path = File.join(ENV["HOME"], '.rbbt/tmp/tmp-ssh_job_inputs/#{inputs_id}')
-job_inputs = Workflow.load_inputs(path, task_info[:inputs], task_info[:input_types])
+job_inputs = wf.tasks[task].load_inputs(path)
 job = wf.job(task, jobname, job_inputs)
       EOF
       script
@@ -199,28 +199,6 @@ job.clean
       end
     end
 
-    #def self.relay_old(workflow, task, jobname, inputs, server, options = {})
-    #  options = Misc.add_defaults options, :search_path => 'user'
-    #  search_path = options[:search_path]
-
-    #  job = workflow.job(task, jobname, inputs)
-
-    #  job.dependencies.each do |dep| 
-    #    dep.produce 
-    #  end
-
-    #  override_dependencies = job.dependencies.collect{|dep| [dep.workflow.to_s, dep.task_name.to_s] * "#" << "=" << Rbbt.identify(dep.path)}
-
-    #  job.dependencies.each do |dep| 
-    #    Step.migrate(dep.path, search_path, :target => server)
-    #  end
-
-    #  remote = RemoteWorkflow.new("ssh://#{server}:#{workflow.to_s}", "#{workflow.to_s}")
-    #  rjob = remote.job(task, jobname, {})
-    #  rjob.override_dependencies = override_dependencies
-    #  rjob.run
-    #end
-
     def self.upload_dependencies(job_list, server, search_path = 'user', produce_dependencies = false)
       server, path = parse_url(server) if server =~ /^ssh:\/\//
 
@@ -229,13 +207,22 @@ job.clean
       all_deps = {}
       if produce_dependencies
         job_list.each do |job|
-          job.dependencies.each do |dep|
+          job.all_dependencies.each do |dep|
             all_deps[dep] ||= []
             all_deps[dep] << job
           end
         end
       end
 
+      job_list.each do |job|
+        job.dependencies.
+          select{|dep| dep.overriden? }.
+          each do |dep|
+            all_deps[dep] ||= []
+            all_deps[dep] << job
+          end
+      end
+
       job_list.each do |job|
         job.input_dependencies.each do |dep|
           all_deps[dep] ||= []
@@ -246,13 +233,16 @@ job.clean
       missing_deps = []
       all_deps.each do |dep,jobs|
         next if dep.done?
+        next if job_list.include?(dep)
         Log.medium "Producing #{dep.workflow}:#{dep.short_path} dependency for #{Misc.fingerprint jobs}"
-        dep.run(true)
+        dep.produce
         missing_deps << dep
       end if produce_dependencies
+
       Step.wait_for_jobs missing_deps
 
-      migrate_dependencies = all_deps.keys.collect{|d| [d] + d.rec_dependencies + d.input_dependencies }.flatten.select{|d| d.done? }.collect{|d| d.path }
+      #migrate_dependencies = all_deps.keys.collect{|d| [d] + d.rec_dependencies + d.input_dependencies }.flatten.select{|d| d.done? }.collect{|d| d.path }
+      migrate_dependencies = all_deps.keys.collect{|d| [d] + d.input_dependencies }.flatten.select{|d| d.done? }.collect{|d| d.path }
       Log.low "Migrating #{migrate_dependencies.length} dependencies from #{Misc.fingerprint job_list} to #{ server }" 
       Step.migrate(migrate_dependencies, search_path, :target => server) if migrate_dependencies.any?
     end
@@ -283,7 +273,13 @@ job.clean
       upload_dependencies(job, server, search_path, options[:produce_dependencies])
       rjob = remote_workflow.job(job.task_name.to_s, job.clean_name, inputs)
 
-      override_dependencies = job.rec_dependencies.select{|dep| dep.done? }.collect{|dep| [dep.workflow.to_s, dep.task_name.to_s] * "#" << "=" << Rbbt.identify(dep.path)}
+      override_dependencies = {}
+      job.rec_dependencies.
+        select{|dep| dep.done? }.
+        collect{|dep| 
+          override_dependencies[[dep.overriden_workflow.to_s, dep.overriden_task.to_s] * "#"] = dep
+        }
+
       rjob.override_dependencies = override_dependencies
 
       rjob.run_type = run_type
@@ -311,18 +307,29 @@ job.clean
         
         workflow_name = job.workflow.to_s
         remote_workflow = RemoteWorkflow.new("ssh://#{server}:#{workflow_name}", "#{workflow_name}")
-        inputs = IndiferentHash.setup(job.recursive_inputs.to_hash).slice(*job.real_inputs.map{|i| i.to_s})
+        inputs = IndiferentHash.setup(job.recursive_inputs.to_hash).slice(*job.non_default_inputs.map{|i|  i.to_s})
         Log.medium "Relaying dependency #{job.workflow}:#{job.short_path} to #{server} (#{inputs.keys * ", "})"
 
-        rjob = remote_workflow.job(job.task_name.to_s, job.clean_name, inputs)
+        rjob = remote_workflow.job(job.task_name.to_s, job.clean_name, job.provided_inputs)
+
+        override_dependencies = {}
+        job.rec_dependencies.
+          select{|dep| dep.done? }.
+          collect{|dep| 
+            dep_key = if dep.overriden_workflow
+                        [dep.overriden_workflow.to_s, dep.overriden_task.to_s] * "#"
+                      else
+                        [dep.workflow.to_s, dep.task_name.to_s] * "#"
+                      end
+            override_dependencies[dep_key] ||= dep
+          }
 
-        override_dependencies = job.rec_dependencies.select{|dep| dep.done? }.collect{|dep| [dep.workflow.to_s, dep.task_name.to_s] * "#" << "=" << Rbbt.identify(dep.path)}
         rjob.override_dependencies = override_dependencies
 
         rjob.run_type = run_type
         rjob.batch_options = batch_options || {}
 
-        rjob.run(true)
+        rjob.run(:noload)
 
         [rjob, job]
       end
@@ -349,7 +356,6 @@ job.clean
 
     def documentation
       @documention ||= IndiferentHash.setup(RemoteWorkflow::SSH.get_json(File.join(url, "documentation")))
-      @documention
     end
 
     def task_info(task)
@@ -382,6 +388,23 @@ job.clean
       end
     end
 
+    def tasks
+      @tasks ||= begin
+                   tasks = Hash.new do |hash,task_name| 
+                     raise Workflow::TaskNotFoundException, "Task #{task_name} not found in workflow #{self.to_s}" unless @task_info.include?(task_name)
+                     info = @task_info[task_name]
+                     task = Task.setup info do |*args|
+                       raise "This is a remote task" 
+                     end
+                     task.name = task_name.to_sym
+                     hash[task_name] = task
+                   end
+
+                   @task_info.keys.each{|k| tasks[k] }
+                   tasks
+                 end
+    end
+
     def task_dependencies
       @task_dependencies ||= Hash.new do |hash,task| 
         hash[task] = if exported_tasks.include? task

data/lib/rbbt/workflow/remote_workflow/remote_step/rest.rb

@@ -1,6 +1,8 @@
 class RemoteStep
   module REST
 
+    DEFAULT_REFRESH_TIME = 2
+
     def get
       params ||= {}
       params = params.merge(:_format => [:string, :boolean, :tsv, :annotations, :array].include?(result_type.to_sym) ? :raw : :json )
@@ -20,7 +22,7 @@ class RemoteStep
     def load
       params = {}
       join unless done? or streaming?
-      raise get_exception if error? or aborted?
+      raise exception if error? or aborted?
       load_res get
     end
 

data/lib/rbbt/workflow/remote_workflow/remote_step/ssh.rb

@@ -1,5 +1,8 @@
 class RemoteStep
   module SSH
+
+    DEFAULT_REFRESH_TIME = 2
+
     attr_accessor :override_dependencies, :run_type, :batch_options, :produce_dependencies
 
     def init_job(cache_type = nil, other_params = {})
@@ -12,9 +15,15 @@ class RemoteStep
       @input_id ||= "inputs-" << rand(100000).to_s
 
       if override_dependencies && override_dependencies.any?
-        override_dependencies.each do |od|
-          name, _sep, value = od.partition("=")
-          inputs[name] = value
+        if Hash === override_dependencies
+          override_dependencies.each do |name,dep|
+            inputs[name] = dep
+          end
+        else
+          override_dependencies.each do |od|
+            name, _sep, value = od.partition("=")
+            inputs[name] = value
+          end
         end
       end
 
@@ -84,7 +93,7 @@ class RemoteStep
       while ! (done? || error? || aborted?)
         sleep 1
       end
-      raise self.get_exception if error?
+      raise self.exception if error?
       self
     end
 
@@ -97,7 +106,7 @@ class RemoteStep
         issue
       else
         produce
-        self.load unless args.first
+        self.load unless stream
       end
     end
 

data/lib/rbbt/workflow/remote_workflow/remote_step.rb

@@ -2,10 +2,10 @@ require 'rbbt/workflow'
 
 class RemoteStep < Step
 
-  attr_accessor :url, :base_url, :task, :base_name, :inputs, :input_types, :result_type, :result_description, :is_exec, :is_stream, :stream_input, :started
+  attr_accessor :url, :base_url, :task, :base_name, :inputs, :input_types, :result_type, :result_description, :is_exec, :is_stream, :stream_input, :started, :refresh_time
 
-  def initialize(base_url, task = nil, base_name = nil, inputs = nil, input_types = nil, result_type = nil, result_description = nil, is_exec = false, is_stream = false, stream_input = nil)
-    @base_url, @task, @base_name, @inputs, @input_types, @result_type, @result_description, @is_exec, @is_stream, @stream_input = base_url, task, base_name, inputs, input_types, result_type, result_description, is_exec, is_stream, stream_input
+  def initialize(base_url, task = nil, base_name = nil, inputs = nil, input_types = nil, result_type = nil, result_description = nil, is_exec = false, is_stream = false, stream_input = nil, refresh_time = nil)
+    @base_url, @task, @base_name, @inputs, @input_types, @result_type, @result_description, @is_exec, @is_stream, @stream_input, @refresh_time = base_url, task, base_name, inputs, input_types, result_type, result_description, is_exec, is_stream, stream_input, refresh_time
     @base_url = "http://" << @base_url unless @base_url =~ /^[a-z]+:\/\//
     @mutex = Mutex.new
     rest = base_url.include?('ssh:') ? false : true
@@ -13,9 +13,11 @@ class RemoteStep < Step
     if rest
       @adaptor = RemoteWorkflow::REST
       self.extend RemoteStep::REST
+      @refresh_time ||= Rbbt::Config.get(:remote_refresh_time, :refresh_time, :ssh_refresh_time, :ssh, :SSH, :default => RemoteStep::REST::DEFAULT_REFRESH_TIME)
     else
       @adaptor = RemoteWorkflow::SSH
       self.extend RemoteStep::SSH
+      @refresh_time ||= Rbbt::Config.get(:remote_refresh_time, :refresh_time, :rest_refresh_time, :rest, :REST, :default => RemoteStep::SSH::DEFAULT_REFRESH_TIME)
     end
 
   end
@@ -26,7 +28,7 @@ class RemoteStep < Step
 
   def cache_file
     begin
-      digest = Misc.obj2digest([base_url, task.to_s, base_name, inputs])
+      digest = Misc.digest([base_url, task.to_s, base_name, inputs])
       Rbbt.var.cache.REST[task.to_s][[clean_name, digest].compact * "."].find
     rescue
       Log.exception $!
@@ -134,7 +136,7 @@ class RemoteStep < Step
     return {:status => :waiting } unless started?
     @done = @info && @info[:status] && (@info[:status].to_sym == :done || @info[:status].to_sym == :error)
 
-    if !@done && (@last_info_time.nil? || (Time.now - @last_info_time) > 0.5)
+    if !@done && (@last_info_time.nil? || (Time.now - @last_info_time) > @refresh_time)
       update = true 
     else
       update = false
@@ -163,8 +165,8 @@ class RemoteStep < Step
     begin
       status = info[:status]
       @done = true if status and status.to_sym == :done
-      Log.low "RemoteStep status '#{status}' #{self.url}"
-      status
+      Log.low "RemoteStep status '#{status}' #{self.url}" if @status != status
+      @status = status
     rescue
       Log.exception $!
       nil
@@ -258,6 +260,14 @@ class RemoteStep < Step
       Misc.consume_stream(res, true) 
     end
 
+    case @adaptor
+    when RemoteWorkflow::REST
+      max_tries = Scout::Config.get :max_tries, :remote_step, :rest, :default => nil
+    when RemoteWorkflow::SSH
+      max_tries = Scout::Config.get :max_tries, :remote_step, :ssh, :default => 10
+    end
+
+    times = 0
     if not (self.done? || self.aborted? || self.error?)
       self.info 
       return self if self.done? || self.aborted? || self.error?
@@ -265,6 +275,8 @@ class RemoteStep < Step
       sleep 1 unless self.done? || self.aborted? || self.error?
       while not (self.done? || self.aborted? || self.error?)
         sleep 3
+        raise "Max tries reached while waiting for remote job: #{Log.fingerprint self}" if times >  max_tries
+        times += 1
       end
     end
 

data/lib/rbbt/workflow/remote_workflow.rb

@@ -1,4 +1,5 @@
 require 'rbbt/workflow'
+require 'rbbt/tsv'
 class RemoteWorkflow
   include Workflow
 
@@ -23,7 +24,7 @@ class RemoteWorkflow
     name
   end
 
-  def __job(task, name = nil, inputs = {})
+  def remote_job(task, name = nil, inputs = {})
     task_info = task_info(task)
     fixed_inputs = {}
     input_types = IndiferentHash.setup(task_info[:input_types])
@@ -32,6 +33,8 @@ class RemoteWorkflow
       k = k.to_sym
       if TSV === v
         fixed_inputs[k] = v.to_s
+      elsif Path === v && Path.step_file?(v)
+        fixed_inputs[k] = v.identify
       else
         next if input_types[k].nil?
         case input_types[k].to_sym
@@ -49,6 +52,8 @@ class RemoteWorkflow
     step
   end
 
+  alias job remote_job
+
   def load_id(id)
     task, name = id.split("/")
     step = RemoteStep.new url, task, nil