rbbt-util 5.33.9 → 5.33.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5c1ff60078b8ea97b0d68c7ec8beb1831889cd0e4ef841c781073dad938924be
-  data.tar.gz: 75dd8c9528cc2015c83b3d106027a5bdc29ba5d9eca53b8024ab19eb824c5273
+  metadata.gz: e5c26a161d8529f19d32ace8b5565042b4baa6fef3d69580f46aadc523e8cc51
+  data.tar.gz: 8ee1305d3a66a55273474da60e84cec3385391ad99c8c87e764fd72eaaf29285
 SHA512:
-  metadata.gz: 60379d7fd62c52f115a198bab61ef24ba2dc4be5c0fba9d4974637afaea9481b99530d0cadbae07b03db92fec75dee1ea1bb23d3fa96906d6cc918794ab0973b
-  data.tar.gz: 7c7ffd9b0847083722e6976f98eb700fef719f7fba7b36914e690538a93de047c7c46d42c0715ac430edc9c392ad5b0f1b0e97de65f812ba2eed3835263dbbe9
+  metadata.gz: 6b9ddf9ff821584ef21f7c30be381965ea9a8cf1cce7e9913cc11c118fa1d31211fe321d72c5d8daabcb14aac5b20df45c527fefa1d142c7a08633c3818b79dd
+  data.tar.gz: c0ab742b23fe87fd7f52bfdfe75690a02566547a34a484912c74766cff70c83634434d5a67a9e665bfdffb28be143afdb20028356befe045e19b14bcc83a21ff

data/lib/rbbt/hpc/batch.rb

@@ -88,8 +88,8 @@ module HPC
     def rbbt_job_exec_cmd(job, options)
 
       jobname  = job.clean_name
-      workflow = job.workflow
-      task     = job.task_name
+      workflow = job.original_workflow || job.workflow
+      task     = job.original_task_name || job.task_name
 
       Misc.add_defaults options, :jobname => jobname
 
@@ -535,8 +535,8 @@ env > #{batch_options[:fenv]}
         :batch_base_dir, :clean_batch_job, :remove_batch_dir, :batch_procpath, :tail, :batch_dependencies, :dry_run,
         :batch_base_dir => File.expand_path(File.join('~/rbbt-batch')) 
 
-      workflow = job.workflow
-      task_name = job.task_name
+      workflow = job.original_workflow ||job.workflow
+      task_name = job.original_task_name || job.task_name
 
       workflows_to_load = job.rec_dependencies.select{|d| Step === d}.collect{|d| d.workflow }.compact.collect(&:to_s) - [workflow.to_s]
 

data/lib/rbbt/hpc/orchestrate/batches.rb

@@ -101,7 +101,9 @@ module HPC
 
       batches.each do |batch|
         job_rules = batch[:jobs].inject(nil) do |acc,job|
-          task_rules = task_specific_rules(rules, job.workflow, job.task_name)
+          workflow = job.original_workflow || job.workflow
+          task_name = job.original_task_name || job.task_name
+          task_rules = task_specific_rules(rules, workflow, task_name)
           acc = accumulate_rules(acc, task_rules.dup)
         end
 
@@ -120,13 +122,16 @@ module HPC
         end
 
         batches.each do |batch|
+          next if batch[:top_level].overriden?
           next unless batch[:rules][:skip]
           batch[:rules].delete :skip
           next if batch[:deps].nil?
 
           if batch[:deps].any?
+            batch_dep_jobs = batch[:top_level].rec_dependencies
             target = batch[:deps].select do |target|
-              (batch[:deps] - [target] - target[:deps]).empty?
+              batch_dep_jobs.include?(target[:top_level]) && # Don't piggyback batches that are an input dependency, only real dependencies
+                (batch[:deps] - [target] - target[:deps]).empty?
             end.first
             next if target.nil?
             target[:jobs] = batch[:jobs] + target[:jobs]

data/lib/rbbt/hpc/orchestrate/chains.rb

@@ -4,8 +4,10 @@ module HPC
       return [] if Symbol === job.overriden
       matches = []
       chains.each do |name, chain|
-        next unless chain[:tasks].include?(job.workflow.to_s)
-        next unless chain[:tasks][job.workflow.to_s].include?(job.task_name.to_s)
+        workflow = job.original_workflow || job.workflow
+        task_name = job.original_task_name || job.task_name
+        next unless chain[:tasks].include?(workflow.to_s)
+        next unless chain[:tasks][workflow.to_s].include?(task_name.to_s)
         matches << name
       end
       matches
@@ -38,46 +40,50 @@ module HPC
     end
 
     def self.job_chains(rules, job)
-      chains = self.parse_chains(rules)
-
-      matches = check_chains(chains, job)
-
-      dependencies = job_dependencies(job)
-
-      job_chains = []
-      new_job_chains = {}
-      dependencies.each do |dep|
-        dep_matches = check_chains(chains, dep)
-        common = matches & dep_matches
-
-        dep_chains = job_chains(rules, dep)
-        found = []
-        dep_chains.each do |match,info|
-          if common.include?(match)
-            found << match
-            new_info = new_job_chains[match] ||= {}
-            new_info[:jobs] ||= []
-            new_info[:jobs].concat info[:jobs]
-            new_info[:top_level] = job
-          else
+      @@job_chains ||= {}
+      @@job_chains[Misc.digest([rules, job.path].inspect)] ||= 
+        begin
+          chains = self.parse_chains(rules)
+
+          matches = check_chains(chains, job)
+
+          dependencies = job_dependencies(job)
+
+          job_chains = []
+          new_job_chains = {}
+          dependencies.each do |dep|
+            dep_matches = check_chains(chains, dep)
+            common = matches & dep_matches
+
+            dep_chains = job_chains(rules, dep)
+            found = []
+            dep_chains.each do |match,info|
+              if common.include?(match)
+                found << match
+                new_info = new_job_chains[match] ||= {}
+                new_info[:jobs] ||= []
+                new_info[:jobs].concat info[:jobs]
+                new_info[:top_level] = job
+              else
+                job_chains << [match, info]
+              end
+            end
+
+            (common - found).each do |match|
+              info = {}
+              info[:jobs] = [job, dep]
+              info[:top_level] = job
+              job_chains << [match, info]
+            end
+          end
+
+          new_job_chains.each do |match,info|
+            info[:jobs].prepend job
             job_chains << [match, info]
           end
-        end
 
-        (common - found).each do |match|
-          info = {}
-          info[:jobs] = [job, dep]
-          info[:top_level] = job
-          job_chains << [match, info]
+          job_chains
         end
-      end
-
-      new_job_chains.each do |match,info|
-        info[:jobs].prepend job
-        job_chains << [match, info]
-      end
-
-      job_chains
     end
 
   end

data/lib/rbbt/hpc/orchestrate.rb

@@ -82,7 +82,7 @@ module HPC
 
         if options[:dry_run]
           puts Log.color(:magenta, "Manifest: ") + Log.color(:blue, job_options[:manifest] * ", ") + " - tasks: #{job_options[:task_cpus] || 1} - time: #{job_options[:time]} - config: #{job_options[:config_keys]}"
-          puts Log.color(:magenta, "Deps: ") + Log.color(:blue, job_options[:batch_dependencies]*", ")
+          puts Log.color(:yellow, "Deps: ") + Log.color(:blue, job_options[:batch_dependencies]*", ")
           puts Log.color(:yellow, "Path: ") + top[:top_level].path
           puts Log.color(:yellow, "Options: ") + job_options.inspect
           batch_ids[top] = top[:top_level].task_signature

data/lib/rbbt/util/misc/inspect.rb

@@ -355,7 +355,7 @@ module Misc
               "<IO:" << obj.path << "--" << mtime_str(obj.path) << ">"
             end
           when (defined? Step and Step)
-            "<IO:" << obj.short_path << ">"
+            "<IO:" << obj.short_path_real << ">"
           when IO
             if obj.respond_to? :filename and obj.filename
               if defined?(Step) && Open.exists?(Step.info_file(obj.filename))

data/lib/rbbt/util/misc/system.rb

@@ -113,9 +113,10 @@ end
   end
 
   def self.is_filename?(string, need_to_exists = true)
+    return false if string.nil?
     return true if defined? Path and Path === string
     return true if string.respond_to? :exists
-    return true if String === string and string.length < 265 and (File.exist?(string) || ! need_to_exists)
+    return true if String === string and string.split("/").select{|p| p.length > 265}.empty? and (! need_to_exists || File.exist?(string))
     return false
   end
 

data/lib/rbbt/workflow/dependencies.rb

@@ -72,9 +72,12 @@ module Workflow
   def setup_override_dependency(dep, workflow, task_name)
     return [] if dep == :skip || dep == 'skip'
     dep = Step === dep ? dep.dup : Workflow.load_step(dep)
+
+    dep.original_workflow ||= dep.workflow if dep.workflow
+    dep.original_task_name ||= dep.task_name if dep.workflow
+
     dep.workflow = workflow
     dep.info[:name] = dep.name
-    dep.original_task_name ||= dep.task_name if dep.workflow
 
 
     begin

data/lib/rbbt/workflow/step/accessor.rb

@@ -96,6 +96,10 @@ class Step
     [task_name, name] * "/"
   end
 
+  def short_path_real
+    [(Symbol === overriden ? overriden : task_name).to_s, name] * "/"
+  end
+
   def workflow_short_path
     return short_path unless workflow
     workflow.to_s + "#" + short_path

data/lib/rbbt/workflow/step/dependencies.rb

@@ -510,7 +510,7 @@ class Step
 
   def overriden?
     return true if @overriden
-    return true if dependencies.select{|dep| dep.overriden? }.any?
+    return true if dependencies && dependencies.select{|dep| dep.overriden? }.any?
     info[:archived_info].each do |f,i|
       next if Symbol === i
       return true if i[:overriden] || i["overriden"]

data/lib/rbbt/workflow/step/save_load_inputs.rb

@@ -21,16 +21,24 @@ module Workflow
         Log.debug "Trying #{ input }: #{file}"
         next unless file and (File.symlink?(file) || file.exists?)
 
-        type = input_types[input]
+        type = orig_type = input_types[input]
 
         type = :io if file.split(".").last == 'as_io'
 
+        type = :io_array if file.split(".").last == 'as_io_array'
+
         type = :step if file.split(".").last == 'as_step'
 
+        type = :step_array if file.split(".").last == 'as_step_array'
+
         type = :step_file if file.split(".").last == 'as_step_file'
 
+        type = :step_file_array if file.split(".").last == 'as_step_file_array'
+
         type = :path if file.split(".").last == 'as_path'
 
+        type = :path_array if file.split(".").last == 'as_path_array'
+
         type = :filename if file.split(".").last == 'as_filename'
 
         type = :nofile if file.split(".").last == 'nofile'
@@ -38,18 +46,34 @@ module Workflow
         case type
         when :nofile
           inputs[input.to_sym]  = Open.realpath(file)
+        when :path_array
+          inputs[input.to_sym]  = Open.read(file).strip.split("\n")
         when :path
-          inputs[input.to_sym]  = Open.read(file).strip
+          inputs[input.to_sym]  = Open.read(file).strip.split("\n").first
+        when :io
+          inputs[input.to_sym] = Open.open(Open.realpath(file)).split("\n")
         when :io
-          inputs[input.to_sym] = Open.open(Open.realpath(file))
+          inputs[input.to_sym] = Open.realpath(file).split("\n").collect{|f| Open.open(f)}
+        when :step_array
+          steps = Open.read(file).strip.split("\n").collect{|path| Workflow.load_step(path) }
+          inputs[input.to_sym] = steps
         when :step
-          inputs[input.to_sym] = Workflow.load_step(Open.read(file).strip)
+          steps = Open.read(file).strip.split("\n").collect{|path| Workflow.load_step(path) }
+          inputs[input.to_sym] = steps.first
         when :step_file
           path = Open.read(file).strip
           path.extend Path
           step_path = path.match(/(.*)\.files/)[1]
           path.resource = Step.new step_path
           inputs[input.to_sym] = path
+        when :step_file_array
+          paths = Open.read(file).split("\n")
+          paths.each do |path| 
+            path.extend Path
+            step_path = path.match(/(.*)\.files/)[1]
+            path.resource = Step.new step_path
+          end
+          inputs[input.to_sym] = paths
         when :file, :binary
           Log.debug "Pointing #{ input } to #{file}"
           if file =~ /\.yaml/
@@ -187,7 +211,6 @@ class Step
   def self.save_input(name, value, type, dir)
     path = File.join(dir, name.to_s)
 
-
     case value
     when Path
       if Step === value.resource
@@ -207,6 +230,24 @@ class Step
       value = value.path
       path = path + '.as_step'
     when Array
+      case value.first
+      when Path
+        if Step === value.first.resource
+          path = path + '.as_step_file_array'
+        else
+          path = path + '.as_path_array'
+        end
+      when String
+        if Misc.is_filename?(value.first, false)
+          path = path + '.as_path_array'
+        end
+      when IO
+        path = path + '.as_io_array'
+      when Step
+        path = path + '.as_step_array'
+        value = value.collect{|s| s.path }
+      end
+
       value = value * "\n"
     end
 
@@ -226,8 +267,8 @@ class Step
   def self.save_job_inputs(job, dir, options = nil)
     options = IndiferentHash.setup options.dup if options
 
-    task_name = Symbol === job.overriden ? job.overriden : job.task_name
-    workflow = job.workflow
+    task_name = job.original_task_name || job.task_name
+    workflow = job.original_workflow || job.workflow
     workflow = Kernel.const_get workflow if String === workflow
     if workflow
       task_info = IndiferentHash.setup(workflow.task_info(task_name))

data/lib/rbbt/workflow/step.rb

@@ -15,7 +15,7 @@ class Step
   attr_accessor :exec
   attr_accessor :relocated
   attr_accessor :result, :mutex, :seen
-  attr_accessor :real_inputs, :original_task_name
+  attr_accessor :real_inputs, :original_task_name, :original_workflow
 
   RBBT_DEBUG_CLEAN = ENV["RBBT_DEBUG_CLEAN"] == 'true'
 

data/share/rbbt_commands/hpc/list

@@ -209,7 +209,9 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
          if j.file(:progress).exists?
            bar = Log::ProgressBar.new 
            bar.load(j.file(:progress).yaml)
-           prog_rep << bar.report_msg.split("·")[0..1]
+           rep = bar.report_msg.split("·")[1]
+           rep = rep.sub(/.*?(\d+%)/, Log.color(:blue,'\1')).sub(/\-.*/,'')
+           prog_rep << [rep]
          end
        end
      end

data/share/rbbt_commands/hpc/orchestrate

@@ -60,5 +60,6 @@ class Step
     end
 end
 
-ARGV.concat ["-W", $slurm_options[:workflows], '--detach'] if $slurm_options[:workflows]
+ARGV.concat ['--detach']
+ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]
 load Rbbt.share.rbbt_commands.workflow.task.find

data/share/rbbt_commands/hpc/tail

@@ -57,6 +57,7 @@ else
   step_path = nil
 end
 
+puts Log.color(:magenta, "Directory: ") + directory if directory
 puts Log.color(:magenta, "Step path: ") + step_path if step_path
 
 HPC::BATCH_MODULE.follow_job directory, true

data/share/rbbt_commands/lsf/list

@@ -209,7 +209,9 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
          if j.file(:progress).exists?
            bar = Log::ProgressBar.new 
            bar.load(j.file(:progress).yaml)
-           prog_rep << bar.report_msg.split("·")[0..1]
+           rep = bar.report_msg.split("·")[1]
+           rep = rep.sub(/.*?(\d+%)/, Log.color(:blue,'\1')).sub(/\-.*/,'')
+           prog_rep << [rep]
          end
        end
      end

data/share/rbbt_commands/lsf/orchestrate

@@ -60,5 +60,6 @@ class Step
     end
 end
 
-ARGV.concat ["-W", $slurm_options[:workflows], '--detach'] if $slurm_options[:workflows]
+ARGV.concat ['--detach']
+ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]
 load Rbbt.share.rbbt_commands.workflow.task.find

data/share/rbbt_commands/lsf/tail

@@ -57,6 +57,7 @@ else
   step_path = nil
 end
 
+puts Log.color(:magenta, "Directory: ") + directory if directory
 puts Log.color(:magenta, "Step path: ") + step_path if step_path
 
 HPC::BATCH_MODULE.follow_job directory, true

data/share/rbbt_commands/slurm/list

@@ -209,7 +209,9 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
          if j.file(:progress).exists?
            bar = Log::ProgressBar.new 
            bar.load(j.file(:progress).yaml)
-           prog_rep << bar.report_msg.split("·")[0..1]
+           rep = bar.report_msg.split("·")[1]
+           rep = rep.sub(/.*?(\d+%)/, Log.color(:blue,'\1')).sub(/\-.*/,'')
+           prog_rep << [rep]
          end
        end
      end

data/share/rbbt_commands/slurm/orchestrate

@@ -60,5 +60,6 @@ class Step
     end
 end
 
-ARGV.concat ["-W", $slurm_options[:workflows], '--detach'] if $slurm_options[:workflows]
+ARGV.concat ['--detach']
+ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]
 load Rbbt.share.rbbt_commands.workflow.task.find

data/share/rbbt_commands/slurm/tail

@@ -57,6 +57,7 @@ else
   step_path = nil
 end
 
+puts Log.color(:magenta, "Directory: ") + directory if directory
 puts Log.color(:magenta, "Step path: ") + step_path if step_path
 
 HPC::BATCH_MODULE.follow_job directory, true

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.33.9
+  version: 5.33.11
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-03-18 00:00:00.000000000 Z
+date: 2022-04-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake