rbbt-util 5.33.6 → 5.33.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7ef9895840235ea220e85895364588fca0c56b1ed8673483474465bfbf94a32b
-  data.tar.gz: 68ba6769881c9118256efd14c8d05ed1ca6f8ab56733800c45c25631016647d0
+  metadata.gz: f4272048a5c86e74b051600f0db895ea42f4ad51509a0de9ce88452d27133746
+  data.tar.gz: f10d19e028de390beb5e2649c7d2f1e0a7754d3235abe3eeba2790946b347f64
 SHA512:
-  metadata.gz: 53c4c81fcfafd8e001f33886323829e230a93731ba23379e1f40358f3bcfae72931d3978d2db3a490b48f22c618462947d5f63a3cc7d135564588d4f6b48aae8
-  data.tar.gz: c53364bbc000b098d4b80afaab6fa8fb16802f9106bfcb8a0aa847b6b8d970e373f102a0fec6021e817a6ebec58d7dc7586c315c9f3aee67481d975d02558abc
+  metadata.gz: 0667eb9d5783d7722c33f72a66a1c406d330d086a37660aa7682006d10fe15860a9e4cbfcb461f17ed15b146fa294a6f6d8158d937884f14dc7952908f63c357
+  data.tar.gz: f3ba25b347bc4a7f7b56ecdbd1a0bef932b646ac030b3d98fe6ddb76f36a6863073f4692747ceef32c42e4344538a7ee19b531a25b16cc2fa208b201887b441e

data/lib/rbbt/hpc/batch.rb

@@ -544,7 +544,7 @@ env > #{batch_options[:fenv]}
         Misc.add_defaults options, 
           :batch_dir => batch_dir, 
           :inputs_dir => File.join(batch_dir, "inputs_dir"),
-          :workflows => workflows_to_load * ","
+          :workflows => workflows_to_load.uniq * ","
 
         options[:procpath_performance] ||= File.join(batch_dir, "procpath##{procpath.gsub(',', '#')}") if procpath
 

data/lib/rbbt/hpc/orchestrate/chains.rb

@@ -1,6 +1,7 @@
 module HPC
   module Orchestration
     def self.check_chains(chains, job)
+      return [] if Symbol === job.overriden
       matches = []
       chains.each do |name, chain|
         next unless chain[:tasks].include?(job.workflow.to_s)
@@ -36,95 +37,6 @@ module HPC
       (job.dependencies + job.input_dependencies).uniq.select{|d| ! d.done? || d.dirty? }
     end
 
-    #def self.job_workload(job)
-    #  workload = []
-    #  heap = []
-    #  heap << job
-    #  while job = heap.pop
-    #    next if job.done?
-    #    workload << job
-    #    heap.concat job_dependencies(job)
-    #    heap.uniq!
-    #  end
-    #  workload.uniq
-    #end
-
-    #def self.top_level_job(jobs)
-    #  top = jobs.select do |job|
-    #    (jobs - job_workload(job)).empty? &&
-    #      (job_workload(job) - jobs).select{|j| (job_workload(j) & jobs).any? }.empty?
-    #  end
-    #  return nil if top.length != 1
-    #  top.first
-    #end
-
-    #def self.job_chains(rules, job)
-    #  workload = job_workload(job)
-    #  chains = parse_chains(rules)
-
-    #  chain_jobs = {}
-    #  workload.each do |job|
-    #    check_chains(chains, job).each do |match|
-    #      chain_jobs[match] ||= []
-    #      chain_jobs[match] << job
-    #    end
-    #  end
-
-    #  job_chains = []
-
-    #  seen = []
-    #  chain_jobs.sort_by{|name,jobs| jobs.length }.reverse.each do |name,jobs|
-    #    remain = jobs - seen
-    #    next unless remain.length > 1
-    #    top_level_job = top_level_job(jobs)
-    #    next if top_level_job.nil?
-    #    job_chains << {:jobs => remain, :rules => chains[name][:rules], :top_level_job => top_level_job}
-    #    seen.concat remain
-    #  end
-
-    #  job_chains
-    #end
-
-    #def self._job_chains(rules, job)
-    #  workload = job_workload(job)
-    #  chains = parse_chains(rules)
-
-    #  matches = check_chains(chains, job)
-
-    #  job_chains = {}
-    #  job.dependencies.each do |dep|
-    #    dep_chains = _job_chains(rules, dep)
-    #    matches.each do |match|
-    #      if dep_chains[match] && dep_chains[match].include?(dep)
-    #        dep_chains[match].prepend job
-    #      end
-    #    end
-    #    job_chains.merge!(dep_chains)
-    #  end
-
-    #  matches.each do |match|
-    #    job_chains[match] ||= [job]
-    #  end
-
-    #  job_chains
-    #end
-
-    #def self.job_chains(rules, job)
-    #  job_chains = self._job_chains(rules, job)
-    #  iif job_chains
-    #  chains = parse_chains(rules)
-
-    #  seen = []
-    #  job_chains.collect do |name,jobs|
-    #    remain = jobs - seen
-    #    next unless remain.length > 1
-    #    top_level_job = top_level_job(jobs)
-    #    next if top_level_job.nil?
-    #    seen.concat remain
-    #    {:jobs => remain, :rules => chains[name][:rules], :top_level_job => top_level_job}
-    #  end.compact
-    #end
-
     def self.job_chains(rules, job)
       chains = self.parse_chains(rules)
 

data/lib/rbbt/hpc/orchestrate.rb

@@ -5,6 +5,19 @@ require 'rbbt/hpc/orchestrate/batches'
 module HPC
   module Orchestration
 
+    def prepare_for_execution(job)
+      rec_dependencies = job.rec_dependencies(true)
+
+      return if rec_dependencies.empty?
+
+      all_deps = rec_dependencies + [job]
+
+      all_deps.each do |dep|
+        Step.prepare_for_execution(dep)
+      end
+
+    end
+
     def orchestrate_job(job, options)
       options.delete "recursive_clean"
       options.delete "clean_task"
@@ -14,6 +27,9 @@ module HPC
       options.delete "detach"
       options.delete "jobname"
 
+      Log.high "Prepare for exec"
+      prepare_for_execution(job)
+
       if options[:orchestration_rules]
         rules = YAML.load(Open.read(options[:orchestration_rules]))
       elsif Rbbt.etc.slurm["default.yaml"].exists?
@@ -24,6 +40,7 @@ module HPC
 
       IndiferentHash.setup(rules)
 
+      Log.high "Compute batches"
       batches = HPC::Orchestration.job_batches(rules, job)
 
       batch_ids = {}

data/lib/rbbt/util/migrate.rb

@@ -1,4 +1,6 @@
 module Rbbt 
+
+  prepare_for_execution(job)
   def self.migrate_source_paths(path, resource = Rbbt, source = nil)
     if source
       lpath, *paths = Misc.ssh_run(source, <<-EOF).split("\n")
@@ -44,6 +46,8 @@ puts resource[path].find(search_path)
     excludes += (options[:exclude] || "").split(/,\s*/)
     excludes_str = excludes.collect{|s| "--exclude '#{s}'" } * " "
 
+    hard_link = options[:hard_link]
+
     other = options[:other] || []
 
     test_str = options[:test] ? '-nv' : ''
@@ -82,6 +86,8 @@ puts resource[path].find(search_path)
         # rsync_args = "-avztAXHP --copy-unsafe-links"
         rsync_args = "-avztAHP --copy-unsafe-links"
 
+        rsync_args << " --link-dest '#{source_path}'" if hard_link && ! options[:source]
+
         cmd = "rsync #{rsync_args} #{test_str} #{files_from_str} #{excludes_str} '#{source_path}' #{target_path} #{other * " "}"
 
         cmd << " && rm -Rf #{source_path}" if options[:delete] && ! options[:files]

data/lib/rbbt/workflow/step/run.rb

@@ -181,8 +181,8 @@ class Step
     #  end
     #end
 
-    Log.high "Some newer files found: #{Misc.fingerprint outdated_time}" if outdated_time.any?
-    Log.high "Some outdated files found: #{Misc.fingerprint outdated_dep}" if outdated_dep.any?
+    Log.medium "Some newer files found: #{Misc.fingerprint outdated_time}" if outdated_time.any?
+    Log.medium "Some outdated files found: #{Misc.fingerprint outdated_dep}" if outdated_dep.any?
 
     outdated_time + outdated_dep
   end

data/share/rbbt_commands/hpc/list

@@ -284,7 +284,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
      puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | tail -n #{tail.to_i}", :no_fail => true).read
    else
      puts Log.color(:magenta, "Log tail: ")
-     puts CMD.cmd(" cat #{File.join(dir, 'std.err')} | grep -v '^[^\\s:]*\\[3.m'  | grep -v -e '^[[:space:]]*$' | tail -n #{tail.to_i} ").read
+     puts CMD.cmd(" cat #{File.join(dir, 'std.err')} | grep -v '^[^\\s:]*\\[3.m'  | grep -v -e '^[[:space:]]*$' | grep -v  \"\\(STDOUT\\|STDERR\\):[[:space:]]*$\" | tail -n #{tail.to_i} ").read
    end
  end
 

data/share/rbbt_commands/lsf/list

@@ -284,7 +284,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
      puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | tail -n #{tail.to_i}", :no_fail => true).read
    else
      puts Log.color(:magenta, "Log tail: ")
-     puts CMD.cmd(" cat #{File.join(dir, 'std.err')} | grep -v '^[^\\s:]*\\[3.m'  | grep -v -e '^[[:space:]]*$' | tail -n #{tail.to_i} ").read
+     puts CMD.cmd(" cat #{File.join(dir, 'std.err')} | grep -v '^[^\\s:]*\\[3.m'  | grep -v -e '^[[:space:]]*$' | grep -v  \"\\(STDOUT\\|STDERR\\):[[:space:]]*$\" | tail -n #{tail.to_i} ").read
    end
  end
 

data/share/rbbt_commands/slurm/list

@@ -284,7 +284,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
      puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | tail -n #{tail.to_i}", :no_fail => true).read
    else
      puts Log.color(:magenta, "Log tail: ")
-     puts CMD.cmd(" cat #{File.join(dir, 'std.err')} | grep -v '^[^\\s:]*\\[3.m'  | grep -v -e '^[[:space:]]*$' | tail -n #{tail.to_i} ").read
+     puts CMD.cmd(" cat #{File.join(dir, 'std.err')} | grep -v '^[^\\s:]*\\[3.m'  | grep -v -e '^[[:space:]]*$' | grep -v  \"\\(STDOUT\\|STDERR\\):[[:space:]]*$\" | tail -n #{tail.to_i} ").read
    end
  end
 

data/share/rbbt_commands/workflow/task

@@ -219,8 +219,8 @@ help = !!options.delete(:help)
 do_fork = !!options.delete(:fork)
 detach = !!options.delete(:detach)
 do_exec = !!options.delete(:exec)
-clean = !!options.delete(:clean)
 clean_task = options.delete(:clean_task)
+clean = !!options.delete(:clean) || clean_task
 override_deps = options.delete(:override_deps)
 recursive_clean = !!options.delete(:recursive_clean)
 out = options.include?(:output) ? File.open(options[:output], 'wb') : STDOUT

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.33.6
+  version: 5.33.7
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-03-09 00:00:00.000000000 Z
+date: 2022-03-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake