rbbt-util 5.30.5 → 5.30.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aa5b1545b1a92199d163bf398ca973c6aa3d78ab37cc724b988e007c87824b3b
-  data.tar.gz: 6cd3ce80ea589fb97a211ebe1d2973dd7c0d3f93b1ccefe10b9ecefbc77ff515
+  metadata.gz: 3a9ed1ed74562cdb81e14bb2b2213398646f76e532bdee48e67f9ff2f5330aad
+  data.tar.gz: e54ecf83a0b4323ffe98f8968f4a65acc2aae2ee5437330f44f14dc17c450ec5
 SHA512:
-  metadata.gz: d8246d2a2686cc38086800fb6c30949bba93c8e78b7feac220b710a4e80f66858f979c5ea41de0beb67aeefd29de045e2315c3b50128f448bfef521ce3d52e15
-  data.tar.gz: 26559322cae3bfb736565e8da0c7e63c3b006c793ab5cda8ec37a04fabfa460001bae6d7da8164ccd310730b81553d1d3b5a1d14cee9e6929793329ce508ec59
+  metadata.gz: 4d2eb887a184e24d78e8c565cefc25f09246d14e0f22d9a9037447806489cb4148fb2d9d3aad4cfe9bca1f256c0ec43afcc343047f20f974e6facea3214f40c0
+  data.tar.gz: 7f593e22911a501e353b6eb49830291a29b038474fd7bfc182e402f255219fc1eaf07311ad334118b43c275abc9ab2a5198d68ccbcdce7f3b4636c38dc698afd

data/lib/rbbt/hpc/orchestrate.rb

@@ -59,7 +59,7 @@ module HPC
       deps
     end
 
-    def self.orchestrate_job(job, options, skip = false, seen = {})
+    def self.orchestrate_job(job, options, skip = false, seen = {}, chains = {})
       return if job.done?
       return unless job.path.split("/")[-4] == "jobs"
       seen[:orchestration_target_job] ||= job
@@ -79,12 +79,15 @@ module HPC
 
       deps = get_job_dependencies(job, job_rules)
 
+      chains[job.path] ||= []
       dep_ids = deps.collect do |dep|
         skip_dep = job_rules["chain_tasks"] &&
           job_rules["chain_tasks"][job.workflow.to_s] && job_rules["chain_tasks"][job.workflow.to_s].include?(job.task_name.to_s)  &&
           job_rules["chain_tasks"][dep.workflow.to_s] && job_rules["chain_tasks"][dep.workflow.to_s].include?(dep.task_name.to_s) 
 
-        deps = seen[dep.path] ||= self.orchestrate_job(dep, options, skip_dep, seen)
+          chains[job.path] << dep if skip_dep
+
+        deps = seen[dep.path] ||= self.orchestrate_job(dep, options, skip_dep, seen, chains)
         if job.canfail_paths.include? dep.path
           [deps].flatten.compact.collect{|id| ['canfail', id] * ":"}
         else
@@ -108,7 +111,21 @@ module HPC
         job_options[:config_keys] = job_options[:config_keys] ? config_keys + "," + job_options[:config_keys] : config_keys
       end
 
-      run_job(job, job_options)
+      manifest = []
+      stack = [job]
+      while dep = stack.pop
+        manifest << dep
+        stack += chains[dep.path] if chains[dep.path]
+      end
+
+      job_options[:manifest] = manifest.uniq.collect{|dep| dep.workflow_short_path}
+
+      if options[:dry_run]
+        puts Log.color(:yellow, "Manifest: ") + Log.color(:blue, job_options[:manifest] * ", ") + " - tasks: #{job_options[:task_cpus] || 1} - time: #{job_options[:time]} - config: #{job_options[:config_keys]}"
+        []
+      else
+        run_job(job, job_options)
+      end
     end
   end
 end

data/lib/rbbt/hpc/slurm.rb

@@ -21,6 +21,8 @@ module HPC
       exclusive        = options.delete :exclusive
       highmem          = options.delete :highmem
 
+      manifest         = options.delete :manifest
+
       queue            = options.delete(:queue) || Rbbt::Config.get('queue', :slurm_queue, :slurm, :SLURM, :default => 'bsc_ls')
       task_cpus        = options.delete(:task_cpus) || 1
       nodes            = options.delete(:nodes) || 1
@@ -258,6 +260,10 @@ EOF
 workflow write_info --recursive --force=false --check_pid "$step_path" slurm_job $SLURM_JOB_ID 
 EOF
 
+      header +=<<-EOF if manifest
+#MANIFEST: #{manifest * ", "}
+      EOF
+
       header +=<<-EOF
 #CMD: #{rbbt_cmd}
       EOF

data/lib/rbbt/persist.rb

@@ -349,7 +349,7 @@ module Persist
 
         Open.notify_write(path)
 
-        return path if persist_options[:no_load]
+        return path if persist_options[:no_load] || type == :path
 
         res
       end

data/lib/rbbt/resource/path.rb

@@ -218,7 +218,14 @@ module Path
 
                      else
                        where = where.to_sym
-                       raise "Did not recognize the 'where' tag: #{where}. Options: #{paths.keys}" unless paths.include? where
+
+                       if paths.include? where
+                         path = paths[where]
+                       elsif where.to_s.include?("/")
+                         path = where.to_s
+                       else
+                         raise "Did not recognize the 'where' tag: #{where}. Options: #{paths.keys}" unless paths.include? where
+                       end
 
                        if where == :lib 
                          libdir = @libdir || Path.caller_lib_dir(caller_lib) || "NOLIBDIR" 
@@ -227,7 +234,6 @@ module Path
                        end
 
                        pwd = FileUtils.pwd
-                       path = paths[where]
                        path = File.join(path, "{PATH}") unless path.include? "PATH}" or path.include? "{BASENAME}"
                        path = path.
                          sub('{PKGDIR}', pkgdir).

data/lib/rbbt/workflow/accessor.rb

@@ -382,8 +382,8 @@ module Workflow
                              else
                                task_info = d[:workflow].task_info(d[:task])
 
-                               inputs = assign_dep_inputs({}, options.merge(d[:inputs] || {}), real_dependencies, task_info) 
-                               d[:workflow]._job(d[:task], d[:jobname], inputs) 
+                               _inputs = assign_dep_inputs({}, options.merge(d[:inputs] || {}), real_dependencies, task_info) 
+                               d[:workflow]._job(d[:task], d[:jobname], _inputs) 
                              end
                        end
                        ComputeDependency.setup(d, compute) if compute
@@ -400,8 +400,8 @@ module Workflow
                          setup_override_dependency(value, dep[:workflow], dep[:task])
                        else
                          task_info = (dep[:task] && dep[:workflow]) ? dep[:workflow].task_info(dep[:task]) : nil
-                         inputs = assign_dep_inputs({}, dep[:inputs], real_dependencies, task_info)
-                         dep = dep[:workflow]._job(dep[:task], dep[:jobname], inputs)
+                         _inputs = assign_dep_inputs({}, dep[:inputs], real_dependencies, task_info)
+                         dep = dep[:workflow]._job(dep[:task], dep[:jobname], _inputs)
                        end
                      end
                    end

data/lib/rbbt/workflow/remote_workflow/remote_step.rb

@@ -315,6 +315,11 @@ class RemoteStep < Step
     end
   end
 
+  def workflow_short_path
+    init_job unless @url
+    [@base_url, @task, @name] * "/"
+  end
+
   def short_path
     init_job unless @url
     [@task, @name] * "/"

data/lib/rbbt/workflow/step.rb

@@ -465,7 +465,10 @@ class Step
     self
   end
 
-  def rec_dependencies(need_run = false, seen = [])
+  #connected = true means that dependency searching ends when a result is done
+  #but dependencies are absent, meanining that the file could have been dropped
+  #in
+  def rec_dependencies(connected = false, seen = [])
 
     # A step result with no info_file means that it was manually
     # placed. In that case, do not consider its dependencies
@@ -480,9 +483,9 @@ class Step
       #next if self.done? && Open.exists?(info_file) && info[:dependencies] && info[:dependencies].select{|task,name,path| path == step.path }.empty?
       next if archived_deps.include? step.path
       next if seen.include? step.path
-      next if self.done? && need_run && ! updatable?
+      next if self.done? && connected && ! updatable?
 
-      r = step.rec_dependencies(need_run, new_dependencies.collect{|d| d.path})
+      r = step.rec_dependencies(connected, new_dependencies.collect{|d| d.path})
       new_dependencies.concat r
       new_dependencies << step
     }

data/lib/rbbt/workflow/step/accessor.rb

@@ -154,10 +154,16 @@ class Step
     @name ||= path.sub(/.*\/#{Regexp.quote task_name.to_s}\/(.*)/, '\1')
   end
 
+
   def short_path
     [task_name, name] * "/"
   end
 
+  def workflow_short_path
+    return short_path unless workflow
+    workflow.to_s + "#" + short_path
+  end
+
   def task_name
     @task_name ||= task.name
   end
@@ -456,7 +462,7 @@ class Step
   end
 
   def dirty_files
-    rec_dependencies = self.rec_dependencies
+    rec_dependencies = self.rec_dependencies(true)
     return [] if rec_dependencies.empty?
     canfail_paths = self.canfail_paths
 

data/lib/rbbt/workflow/step/run.rb

@@ -112,23 +112,27 @@ class Step
   end
 
   def updatable?
-    (ENV["RBBT_UPDATE_ALL_JOBS"] == 'true' || ( ENV["RBBT_UPDATE"] == "true" && Open.exists?(info_file)) && status != :noinfo && ! (relocated? && done?)) || (ENV["RBBT_UPDATE"] && ! (done? && ! Open.exists?(info_file)))
+    return true if ENV["RBBT_UPDATE_ALL_JOBS"] == 'true'
+    return false unless ENV["RBBT_UPDATE"] == "true"
+    return false unless Open.exists?(info_file)
+    return true if status != :noinfo && ! (relocated? && done?)
+    false
   end
 
   def dependency_checks
     return [] if ENV["RBBT_UPDATE"] != "true"
 
-    rec_dependencies.
+    rec_dependencies(true).
       reject{|dependency| (defined?(WorkflowRemoteClient) && WorkflowRemoteClient::RemoteStep === dependency) || Open.remote?(dependency.path) }.
       reject{|dependency| dependency.error? }.
       #select{|dependency| Open.exists?(dependency.path) || ((Open.exists?(dependency.info_file) && (dependency.status == :cleaned) || dependency.status == :waiting)) }.
-      select{|dependency| dependency.updatable? }.
+      #select{|dependency| dependency.updatable? }.
       collect{|dependency| Workflow.relocate_dependency(self, dependency)}
   end
 
   def input_checks
-    (inputs.select{|i| Step === i } + inputs.select{|i| Path === i && Step === i.resource}.collect{|i| i.resource}).
-      select{|dependency| dependency.updatable? }
+    (inputs.select{|i| Step === i } + inputs.select{|i| Path === i && Step === i.resource}.collect{|i| i.resource})
+      #select{|dependency| dependency.updatable? }
   end
 
   def checks
@@ -153,7 +157,8 @@ class Step
     canfail_paths = self.canfail_paths
     this_mtime = Open.mtime(self.path) if Open.exists?(self.path)
 
-    outdated_time = checks.select{|dep| dep.updatable? && dep.done? && Persist.newer?(path, dep.path) }
+    #outdated_time = checks.select{|dep| dep.updatable? && dep.done? && Persist.newer?(path, dep.path) }
+    outdated_time = checks.select{|dep| dep.done? && Persist.newer?(path, dep.path) }
     outdated_dep = checks.reject{|dep| dep.done? || (dep.error? && ! dep.recoverable_error? && canfail_paths.include?(dep.path)) }
 
     #checks.each do |dep| 

data/lib/rbbt/workflow/usage.rb

@@ -21,7 +21,7 @@ module Task
 
     selects = []
     if inputs.any?
-      inputs.zip(input_types.values_at(*inputs)).select{|i,t| t.to_sym == :select and input_options[i][:select_options] }.each{|i,t| selects << [i, input_options[i][:select_options]]  }
+      inputs.zip(input_types.values_at(*inputs)).select{|i,t| t.to_sym == :select && input_options[i] && input_options[i][:select_options] }.each{|i,t| selects << [i, input_options[i][:select_options]]  }
       puts SOPT.input_doc(inputs, input_types, input_descriptions, input_defaults, true)
       puts
     end

data/lib/rbbt/workflow/util/provenance.rb

@@ -91,7 +91,14 @@ class Step
         if expand_repeats
           str << Log.color(:green, Log.uncolor(prov_report(dep, offset+1, task)))
         else
-          str << Log.color(:green, " " * (offset + 1) + Log.uncolor(prov_report_msg(dep.status, dep.info[:name], dep.path, dep.info)))
+          info = dep.info  || {}
+          status = info[:status] || :missing
+          status = "remote" if Open.remote?(path) || Open.ssh?(path)
+          name = info[:name] || File.basename(path)
+          status = :unsync if status == :done and not Open.exist?(path)
+          status = :notfound if status == :noinfo and not Open.exist?(path)
+
+          str << Log.color(status == :notfound ? :blue : :green, " " * (offset + 1) + Log.uncolor(prov_report_msg(status, name, path, info)))
         end
       end
     end if step.dependencies

data/share/rbbt_commands/slurm/list

@@ -70,6 +70,13 @@ workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
   cmd = nil
  end
 
+ if m = Open.read(fcmd).match(/#MANIFEST: (.*)/)
+  manifest = m[1]
+ else
+  manifest = nil
+ end
+
+
  if m = Open.read(fcmd).match(/# Run command\n(.*?)\n/im)
   exe = m[1].sub('step_path=$(','')
  else
@@ -145,6 +152,7 @@ workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
  puts Log.color :blue, dir
  puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.slurm')).to_s
  puts Log.color(:magenta, "Started: ") << File.ctime(File.join(dir, 'std.err')).to_s if File.exist?(File.join(dir, 'std.err'))
+ puts Log.color(:magenta, "Manifest: ") << Log.color(:yellow, manifest) 
  puts Log.color(:magenta, "Done: ") << File.mtime(File.join(dir, 'exit.status')).to_s if File.exist?(File.join(dir, 'exit.status'))
  puts Log.color(:magenta, "Exec: ") << (exe || "Missing")
  puts Log.color(:magenta, "CMD: ") << (Log.color(:yellow, cmd) || "Missing")

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.30.5
+  version: 5.30.6
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-02-07 00:00:00.000000000 Z
+date: 2021-02-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake