rbbt-util 5.35.4 → 5.37.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b3a214b340c407e61c0cc227b8872c4b0c71463affcf6bf84f9d4ed66486435c
-  data.tar.gz: d62735c060afe4507cc69363280d76e24c88ed89063531e526fe5a03ecfd6386
+  metadata.gz: 281f115df5158fada081c65b87e292e69e4f378b8353d6e2542b3c45db0c03f1
+  data.tar.gz: e9801707391c731a2a4b4ead8156cff82fdf30c4e769a8bed34d1d77185b447a
 SHA512:
-  metadata.gz: f8fd64ecb9abc0a5bf519fa294196e8b1082a4ef358106806bc7c18dadfb361bd22b2b14aca96f3dd32c276461fa6616d3613a7de7aca87ceaddc7db74d57e23
-  data.tar.gz: baa4182336b92c09eead8fed9a28b2bc6d84aea7e2e9e98c4597ae3373ac31ff27feb62e1dbd73817ac69f7986a00ceb2bf404d38fb739919882fea684053bfa
+  metadata.gz: 3da2876b28c245b27f32b6dbd00188bdeea498fb118ccab048fb5d72925548e5e89926dea6b6316573e9d95a570c07983109e0e09de92360ec89ab2aec088439
+  data.tar.gz: 0166d47c3198bfc4c636a1f47ac9be1d2a64b8d97a2836166b02d1ac17d46c2b835ad210953da586f15a98f8593e18f6010ebc07220e95da149dcabec44b33a9

data/lib/rbbt/persist.rb

@@ -44,6 +44,8 @@ module Persist
   end
 
   def self.is_persisted?(path, persist_options = {})
+    return true if Open.remote?(path)
+    return true if Open.ssh?(path)
     return false if not Open.exists? path
     return false if TrueClass === persist_options[:update]
 

data/lib/rbbt/resource/path.rb

@@ -319,7 +319,8 @@ module Path
 
 
   def open(options = {}, &block)
-    Open.open(self.produce.find, options, &block)
+    file = Open.remote?(self) || Open.ssh?(self) ? self : self.produce.find
+    Open.open(file, options, &block)
   end
 
   def to_s
@@ -331,7 +332,16 @@ module Path
   end
 
   def tsv(*args)
-    TSV.open(self.produce, *args)
+    begin
+      path = self.produce
+    rescue Resource::ResourceNotFound => e
+      begin
+        path = self.set_extension('tsv').produce
+      rescue Resource::ResourceNotFound 
+        raise e
+      end
+    end
+    TSV.open(path, *args)
   end
 
   def tsv_options(options = {})

data/lib/rbbt/util/concurrency/processes/socket.rb

@@ -28,6 +28,7 @@ class RbbtProcessQueue
 
 
     def dump(obj, stream)
+      obj.concurrent_stream = nil if obj.respond_to?(:concurrent_stream)
       case obj
       when Annotated
         payload = @serializer.dump(obj)

data/lib/rbbt/util/concurrency/processes/worker.rb

@@ -65,11 +65,7 @@ class RbbtProcessQueue
           Log.high "Worker #{Process.pid} leaving"
         rescue Exception
           Log.high "Worker #{Process.pid} had exception: #{$!.message}"
-          Log.exception $!
-          begin
-            @callback_queue.push($!) if @callback_queue
-          rescue
-          end
+          @callback_queue.push($!) if @callback_queue
           Kernel.exit! -1
         ensure
           @callback_queue.close_write if @callback_queue 

data/lib/rbbt/util/concurrency/processes.rb

@@ -180,13 +180,15 @@ class RbbtProcessQueue
         rescue Exception
           Log.low "Process monitor exception [#{Process.pid}]: #{$!.message}"
           processes.each{|p| p.abort_and_join}
-          Log.low "Processes aborted #{Process.pid}"
+          Log.low "Processes aborted for monitor #{Process.pid}"
           processes.clear
 
+          @manager_thread.report_on_exception = false
           @manager_thread.raise $! if @manager_thread.alive?
           raise Aborted, "Aborted monitor thread with exception"
         end
       end
+      @monitor_thread.report_on_exception = false
 
       RbbtSemaphore.post_semaphore(@sem)
 
@@ -215,33 +217,36 @@ class RbbtProcessQueue
     init_master
 
     RbbtSemaphore.synchronize(@sem) do
-    @callback_thread = Thread.new do
-      begin
-        loop do
-          p = @callback_queue.pop unless @callback_queue.cleaned
+      @callback_thread = Thread.new do
+        begin
+          loop do
+            p = @callback_queue.pop unless @callback_queue.cleaned
 
-          if Exception === p or (Array === p and Exception === p.first)
-            e = Array === p ? p.first : p
-            Log.low "Callback recieved exception from worker: #{e.message}" unless Aborted === e or ClosedStream === e
-            raise e 
-          end
+            if Exception === p or (Array === p and Exception === p.first)
+              e = Array === p ? p.first : p
+              Log.low "Callback recieved exception from worker: #{e.message}" unless Aborted === e or ClosedStream === e
+              raise e 
+            end
 
-          if @callback.arity == 0
-            @callback.call
-          else
-            @callback.call p
+            if @callback.arity == 0
+              @callback.call
+            else
+              @callback.call p
+            end
           end
+        rescue ClosedStream
+          Log.low "Callback thread closing"
+        rescue Aborted
+          Log.low "Callback thread aborted"
+          raise $!
+        rescue Exception
+          Log.low "Exception captured in callback: #{$!.message}"
+          raise $!
         end
-      rescue ClosedStream
-        Log.low "Callback thread closing"
-      rescue Aborted
-        Log.low "Callback thread aborted"
-        raise $!
-      rescue Exception
-        Log.low "Exception captured in callback: #{$!.message}"
-        raise $!
-      end
-    end if @callback_queue
+      end if @callback_queue
+
+      @callback_thread.report_on_exception = false
+      
     end
 
   end

data/lib/rbbt/util/misc/inspect.rb

@@ -343,9 +343,9 @@ module Misc
             end
           when Array
             if obj.length > HASH2MD5_MAX_ARRAY_LENGTH
-              "[" << sample_large_obj(obj, HASH2MD5_MAX_ARRAY_LENGTH).collect{|v| obj2str(v)} * "," << "]"
+              "[" << sample_large_obj(obj, HASH2MD5_MAX_ARRAY_LENGTH).collect{|v| obj2str(v.nil? ? "" : v) } * "," << "]"
             else
-              "[" << obj.collect{|v| obj2str(v) } * "," << "]"
+              "[" << obj.collect{|v| obj2str(v.nil? ? "" : v) } * "," << "]"
             end
           when TSV::Parser
             remove_long_items(obj)

data/lib/rbbt/util/python.rb

@@ -59,12 +59,12 @@ module RbbtPython
   end
 
   def self.get_class(module_name, class_name)
-    save_module_name = module_name.gsub(".", "_")
+    save_module_name = module_name.to_s.gsub(".", "_")
     RbbtPython.pyimport(module_name, as: save_module_name)
     RbbtPython.send(save_module_name).send(class_name)
   end
 
-  def self.class_new_obj(module_name, class_name, args)
+  def self.class_new_obj(module_name, class_name, args={})
     RbbtPython.get_class(module_name, class_name).new(**args)
   end
 

data/lib/rbbt/workflow/definition.rb

@@ -175,6 +175,8 @@ module Workflow
 
     tasks[name] = task
     task_dependencies[name] = consume_dependencies
+
+    task
   end
 
   def unexport(*names)

data/lib/rbbt/workflow/dependencies.rb

@@ -72,6 +72,7 @@ module Workflow
   def setup_override_dependency(dep, workflow, task_name)
     return [] if dep == :skip || dep == 'skip'
 
+    unlocated = unlocated_override?(dep)
     dep = Workflow.load_step(dep) if not Step === dep
 
     dep.original_workflow ||= dep.workflow if dep.workflow
@@ -91,7 +92,7 @@ module Workflow
     end
 
     dep.task_name = task_name
-    dep.overriden = dep.original_task_name.to_sym if dep.original_task_name 
+    dep.overriden = dep.original_task_name.to_sym if dep.original_task_name && dep.original_task_name.to_s != task_name.to_s || ! unlocated
 
     dep.extend step_module
 
@@ -169,6 +170,16 @@ module Workflow
                    else
 
                      compute = options[:compute] if options
+                     if options && options[:canfail]
+                       compute = case compute
+                                 when nil
+                                   :canfail
+                                 when Array
+                                   compute + [:canfail]
+                                 else
+                                   [compute, :canfail]
+                                 end
+                     end
 
                      all_d = (real_dependencies + real_dependencies.flatten.collect{|d| d.rec_dependencies} ).flatten.compact.uniq
 
@@ -207,6 +218,16 @@ module Workflow
 
                        options = {} if options.nil?
                        compute = options[:compute]
+                       if options[:canfail]
+                         compute = case compute
+                                   when nil
+                                     :canfail
+                                   when Array
+                                     compute + [:canfail]
+                                   else
+                                     [compute, :canfail]
+                                   end
+                       end
 
                        options = IndiferentHash.setup(options.dup)
                        dep = dependency.call jobname, _inputs.merge(options), real_dependencies
@@ -228,7 +249,8 @@ module Workflow
                                  task_info = d[:workflow].task_info(d[:task])
 
                                  _inputs = assign_dep_inputs({}, options.merge(d[:inputs] || {}), real_dependencies, task_info) 
-                                 job = d[:workflow]._job(d[:task], d[:jobname], _inputs) 
+                                 _jobname = d.include?(:jobname) ? d[:jobname] : jobname
+                                 job = d[:workflow]._job(d[:task], _jobname, _inputs) 
                                  overriden = true if TrueClass === job.overriden && (d.nil? || ! d[:not_overriden])
                                  job
                                end
@@ -250,8 +272,8 @@ module Workflow
                        else
                          task_info = (dep[:task] && dep[:workflow]) ? dep[:workflow].task_info(dep[:task]) : nil
                          _inputs = assign_dep_inputs({}, dep[:inputs], real_dependencies, task_info)
-                         job = dep[:workflow]._job(dep[:task], dep[:jobname], _inputs)
-                         job = d[:workflow]._job(d[:task], d[:jobname], _inputs) 
+                         _jobname = dep.include?(:jobname) ? dep[:jobname] : jobname
+                         job = dep[:workflow]._job(dep[:task], _jobname, _inputs)
                          overriden = true if TrueClass === job.overriden && (d.nil? || ! d[:not_overriden])
                          job
                        end

data/lib/rbbt/workflow/doc.rb

@@ -37,6 +37,7 @@ module Workflow
   def documentation_markdown
     return "" if @libdir.nil?
     file = @libdir['workflow.md'].find
+    file = @libdir['README.md'].find unless file.exists?
     if file.exists?
       file.read
     else

data/lib/rbbt/workflow/remote_workflow/driver/ssh.rb

@@ -1,4 +1,6 @@
 class RemoteWorkflow
+  RBBT_DEBUG_REMOTE_JSON = ENV["RBBT_DEBUG_REMOTE_JSON"] == 'true'
+
   module SSH
     #def self.run(server, script)
     #  Log.debug "Run ssh script in #{server}:\n#{script}"
@@ -21,9 +23,16 @@ class RemoteWorkflow
 
       workflow, task, job, *rest = path.split("/")
 
+      workflow_name = begin
+                        wf = Kernel.const_get(workflow) if String === workflow && ! workflow.empty?
+                        wf.respond_to?(:complete_name) ? (wf.complete_name || workflow) : workflow
+                      rescue
+                        workflow
+                      end
+
       script =<<-EOF
 require 'rbbt/workflow'
-wf = Workflow.require_workflow "#{workflow}"
+wf = Workflow.require_workflow "#{workflow_name}"
       EOF
 
       case task
@@ -86,7 +95,7 @@ STDOUT.write res.to_json
       EOF
 
       json = Misc.ssh_run(server, script)
-      Log.debug "JSON (#{ url }): #{json}"
+      Log.debug "JSON (#{ url }): #{json}" if RBBT_DEBUG_REMOTE_JSON
       JSON.parse(json)
     end
 
@@ -172,11 +181,18 @@ job.clean
     def self.upload_inputs(server, inputs, input_types, input_id)
       TmpFile.with_file do |dir|
         if Step.save_inputs(inputs, input_types, dir)
-          Dir.glob(File.join(dir, "*.as_step")).each do |file|
-            path = Open.read(file).strip
-            new = Step.migrate(path, :user, :target => server)
-            Open.write(file, new)
-          end
+          # Dir.glob(File.join(dir, "*.as_step")).each do |file|
+          #   Log.medium "Migrating Step input #{file} #{ server }" 
+          #   path = Open.read(file).strip
+          #   new = Step.migrate(path, :user, :target => server)
+          #   Open.write(file, new)
+          # end
+
+          files = Dir.glob(File.join(dir, "*.as_step"))
+          paths = files.collect{|f| Open.read(f).strip }
+          new = Step.migrate(paths, :user, :target => server)
+          files.zip(new).each{|file,new| Open.write(file, new) }
+
           CMD.cmd_log("ssh '#{server}' mkdir -p .rbbt/tmp/tmp-ssh_job_inputs/; scp -r '#{dir}' #{server}:.rbbt/tmp/tmp-ssh_job_inputs/#{input_id}")
         end
       end
@@ -204,31 +220,57 @@ job.clean
     #  rjob.run
     #end
 
+    def self.upload_dependencies(job, server, search_path = 'user', produce_dependencies = false)
+      server, path = parse_url(server) if server =~ /^ssh:\/\//
+      job.dependencies.each do |dep|
+        Log.medium "Producing #{dep.workflow}:#{dep.short_path} dependency for #{job.workflow}:#{job.short_path}"
+        dep.produce
+      end if produce_dependencies
+
+      job.input_dependencies.each do |dep|
+        Log.medium "Producing #{dep.workflow}:#{dep.short_path} dependency for #{job.workflow}:#{job.short_path}"
+        dep.produce
+      end 
+
+      migrate_dependencies = job.rec_dependencies.select{|d| d.done? }.collect{|d| d.path }
+      migrate_dependencies += job.input_dependencies.select{|d| d.done? }.collect{|d| d.path }
+      Log.medium "Migrating #{migrate_dependencies.length} dependencies from #{job.path} to #{ server }" 
+      Step.migrate(migrate_dependencies, search_path, :target => server) if migrate_dependencies.any?
+    end
+
+    def self.missing_dep_inputs(job)
+      inputs = job.inputs.to_hash.slice(*job.real_inputs.map{|i| i.to_s})
+      job.dependencies.each do |dep|
+        next if dep.done?
+        iif [dep, dep.inputs, dep.real_inputs]
+        inputs = dep.inputs.to_hash.slice(*dep.real_inputs.map{|i| i.to_s}).merge(inputs)
+        inputs = missing_dep_inputs(dep).merge(inputs)
+      end
+      inputs
+    end
+
     def self.relay_job(job, server, options = {})
-      migrate, produce, produce_dependencies, search_path = Misc.process_options options.dup,
-        :migrate, :produce, :produce_dependencies, :search_path
+      migrate, produce, produce_dependencies, search_path, run_type, slurm_options = Misc.process_options options.dup,
+        :migrate, :produce, :produce_dependencies, :search_path, :run_type, :slurm_options
 
       search_path ||= 'user'
 
       produce = true if migrate
 
       workflow_name = job.workflow.to_s
-      inputs = job.recursive_inputs.to_hash
-
-      job.dependencies.each do |dep|
-        dep.produce 
-      end if options[:produce_dependencies]
-
-      job.rec_dependencies.each do |dep|
-        Step.migrate(dep.path, search_path, :target => server) if dep.done?
-      end
+      remote_workflow = RemoteWorkflow.new("ssh://#{server}:#{workflow_name}", "#{workflow_name}")
+      inputs = job.recursive_inputs.to_hash.slice(*job.real_inputs.map{|i| i.to_s})
+      Log.medium "Relaying dependency #{job.workflow}:#{job.short_path} to #{server} (#{inputs.keys * ", "})"
 
-      remote_workflow = RemoteWorkflow.new("ssh://#{server}:#{job.workflow.to_s}", "#{job.workflow.to_s}")
+      upload_dependencies(job, server, search_path, options[:produce_dependencies])
       rjob = remote_workflow.job(job.task_name.to_s, job.clean_name, inputs)
 
       override_dependencies = job.rec_dependencies.select{|dep| dep.done? }.collect{|dep| [dep.workflow.to_s, dep.task_name.to_s] * "#" << "=" << Rbbt.identify(dep.path)}
       rjob.override_dependencies = override_dependencies
 
+      rjob.run_type = run_type
+      rjob.slurm_options = slurm_options || {}
+
       if options[:migrate]
         rjob.produce
         Step.migrate(Rbbt.identify(job.path), 'user', :source => server) 
@@ -255,7 +297,7 @@ job.clean
       @task_info ||= IndiferentHash.setup({})
 
       if @task_info[task].nil?
-        task_info = RemoteWorkflow::SSH.get_json(File.join(@base_url, task.to_s))
+        task_info = RemoteWorkflow::SSH.get_json(File.join(@base_url || @url, task.to_s))
         task_info = RemoteWorkflow::SSH.fix_hash(task_info)
 
         task_info[:result_type] = task_info[:result_type].to_sym if task_info[:result_type]

data/lib/rbbt/workflow/remote_workflow/remote_step/ssh.rb

@@ -1,6 +1,6 @@
 class RemoteStep
   module SSH
-    attr_accessor :override_dependencies
+    attr_accessor :override_dependencies, :run_type, :slurm_options
 
     def init_job(cache_type = nil, other_params = {})
       return self if @url
@@ -52,15 +52,31 @@ class RemoteStep
     end
     
     def _run
+      RemoteWorkflow::SSH.upload_dependencies(self, @server)
       RemoteWorkflow::SSH.run_job(File.join(base_url, task.to_s), @input_id, @base_name)
     end
 
+    def _run_slurm
+      RemoteWorkflow::SSH.run_slurm_job(File.join(base_url, task.to_s), @input_id, @base_name, @slurm_options || {})
+    end
+
+    def _orchestrate_slurm
+      RemoteWorkflow::SSH.orchestrate_slurm_job(File.join(base_url, task.to_s), @input_id, @base_name, @slurm_options || {})
+    end
+
     def produce(*args)
       input_types = {}
       init_job
-      @remote_path = _run
+      @remote_path = case @run_type
+                     when 'run', :run, nil
+                       _run
+                     when 'slurm', :slurm
+                       _run_slurm
+                     when 'orchestrate', :orchestrate
+                       _orchestrate_slurm
+                     end
       @started = true
-      while ! (done? || error?)
+      while ! (done? || error? || aborted?)
         sleep 1
       end
       raise self.get_exception if error?
@@ -82,6 +98,18 @@ class RemoteStep
       _restart
     end
 
+    def abort
+      Log.warn "not implemented RemoteWorkflow::SSH.abort(@url, @input_id, @base_name)"
+    end
+
+    def input_dependencies
+      @input_dependencies ||= inputs.values.flatten.
+        select{|i| Step === i || (defined?(RemoteStep) && RemoteStep === i) } + 
+        inputs.values.flatten.
+        select{|dep| Path === dep && Step === dep.resource }.
+        select{|dep| ! dep.resource.started? }. # Ignore input_deps already started
+        collect{|dep| dep.resource }
+    end
   end
 end
 

data/lib/rbbt/workflow/remote_workflow/remote_step.rb

@@ -163,6 +163,7 @@ class RemoteStep < Step
     begin
       status = info[:status]
       @done = true if status and status.to_sym == :done
+      Log.low "RemoteStep status '#{status}' #{self.url}"
       status
     rescue
       Log.exception $!

data/lib/rbbt/workflow/step/accessor.rb

@@ -1,4 +1,12 @@
 class Step
+  attr_accessor :clean_name, :path, :task, :workflow, :inputs, :dependencies, :bindings
+  attr_accessor :task_name, :overriden
+  attr_accessor :pid
+  attr_accessor :exec
+  attr_accessor :relocated
+  attr_accessor :result, :mutex, :seen
+  attr_accessor :real_inputs, :original_task_name, :original_workflow
+
 
   INFO_SERIALIZER = begin
                        if ENV["RBBT_INFO_SERIALIZER"]
@@ -86,12 +94,45 @@ class Step
     end
   end
 
+  def task_name
+    @task_name ||= begin
+                     if @task.respond_to?(:name)
+                     
+                     @task.name
+                     else
+                       @path.split("/")[-2]
+                     end
+                   end
+  end
+
+  def result_type
+    @result_type ||= if @task.respond_to?(:result_type)
+                       @task.result_type || info[:result_type]
+                     else
+                       info[:result_type]
+                     end
+  end
+
+  def result_type=(type)
+    @result_type = type
+  end
+
+  def result_description
+    @result_description ||= if @task.respond_to?(:result_description)
+                              @task.result_description
+                            else
+                              nil
+                            end
+  end
+
+  def result_description=(description)
+    @result_description = description
+  end
 
   def name
     @name ||= path.sub(/.*\/#{Regexp.quote task_name.to_s}\/(.*)/, '\1')
   end
 
-
   def short_path
     [task_name, name] * "/"
   end
@@ -105,10 +146,6 @@ class Step
     workflow.to_s + "#" + short_path
   end
 
-  def task_name
-    @task_name ||= task.name
-  end
-
   def task_signature
     [workflow.to_s, task_name] * "#"
   end