rbbt-util 5.28.11 → 5.28.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6ae5e7a2e944e3aabababe528af35f6d0b9002db33c661cf75dd3f6f49bd1b94
-  data.tar.gz: 94a6ec3c4a32e3632d59872c17b449a5c6be624cf999b2bb3469647933a0ba28
+  metadata.gz: 889338e97d8b2f2467dcbbb7d7ecfb1a4904a317702c2b7c7164757efd1f55dc
+  data.tar.gz: 0e71e707ea5d0bab991aaf25f9dd5bf1d371d36ed6267694934aab92218ba293
 SHA512:
-  metadata.gz: 433ee5aa750efe1f609d6ee37a30b4438fc512719de2a7062d8696e2e1f15dec5caa33e2e63dce2ea7082d9f741fadf3314991a6a5e4a30d770f9fe6c7a04207
-  data.tar.gz: d16905de756df32abacd314b8793d4792c9852d76267be7ef71b6c5e0ae4e08f8b9ce04f34f2ceba3cd36ed8cda18966e737a893439b2451a9cde43a334afcb6
+  metadata.gz: 173386b238dda5361b1c473c90708b783736f3726f868e86dd64f732d3e9474fc9987b133fdfc3d19538ccc19b31eb2f91ae29fa1fedaf150e5d46619ef69c76
+  data.tar.gz: 306f1058d7248387a7887bed09c4bd21541e6f52462de6b17d2d4b04bb73ebe4f0cdb3a9245e1a24f61fec252587d3b0669623d8544a6fc0d250eeebdad8c47e

data/lib/rbbt/hpc.rb

@@ -122,7 +122,7 @@ module Marenostrum
 module load java
 
 # Calculate max available memory
-let "MAX_MEMORY=$SLURM_MEM_PER_CPU * $SLURM_CPUS_ON_NODE"
+let "MAX_MEMORY=$SLURM_MEM_PER_CPU * $SLURM_CPUS_PER_TASK"
       EOF
 
 
@@ -172,7 +172,7 @@ cp ~/.rbbt/etc/environment $CONTAINER_DIR/.rbbt/etc/
 # Set search_paths
 echo "singularity: /singularity_opt/{PKGDIR}/{TOPLEVEL}/{SUBPATH}" > $CONTAINER_DIR/.rbbt/etc/search_paths
 echo "rbbt_user: /home/rbbt/.rbbt/{TOPLEVEL}/{SUBPATH}" >> $CONTAINER_DIR/.rbbt/etc/search_paths
-echo "home: $CONTAINER_DIR/home/{TOPLEVEL}/{SUBPATH}" >> $CONTAINER_DIR/.rbbt/etc/search_paths
+echo "outside_home: $CONTAINER_DIR/home/{TOPLEVEL}/{SUBPATH}" >> $CONTAINER_DIR/.rbbt/etc/search_paths
 echo "group_projects: #{projects_group_dir}/{PKGDIR}/{TOPLEVEL}/{SUBPATH}" >> $CONTAINER_DIR/.rbbt/etc/search_paths
 echo "group_scratch: #{scratch_group_dir}/{PKGDIR}/{TOPLEVEL}/{SUBPATH}" >> $CONTAINER_DIR/.rbbt/etc/search_paths
 echo "user_projects: #{projects_group_dir}/#{user}/{PKGDIR}/{TOPLEVEL}/{SUBPATH}" >> $CONTAINER_DIR/.rbbt/etc/search_paths

data/lib/rbbt/persist.rb

@@ -31,8 +31,8 @@ module Persist
     path = path.find if Path === path
     file = file.find if Path === file
     if by_link
-      patht = File.lstat(path).mtime
-      filet = File.lstat(file).mtime
+      patht = File.exists?(path) ? File.lstat(path).mtime : nil
+      filet = File.exists?(file) ? File.lstat(file).mtime : nil
     else
       patht = Open.mtime(path)
       filet = Open.mtime(file)

data/lib/rbbt/resource/path.rb

@@ -297,7 +297,7 @@ module Path
 
     raise "No resource defined to produce file: #{ self }" if resource.nil?
 
-    resource.produce self, force
+    resource.produce self, force if Resource === resource
 
     self
   end

data/lib/rbbt/util/misc/indiferent_hash.rb

@@ -61,6 +61,14 @@ module IndiferentHash
       super(key)
     end
   end
+
+  def clean_version
+    clean = {}
+    each do |k,v|
+      clean[k.to_s] = v unless clean.include? k.to_s
+    end
+    clean
+  end
 end
 
 module CaseInsensitiveHash

data/lib/rbbt/workflow/step.rb

@@ -13,6 +13,8 @@ class Step
   attr_accessor :relocated
   attr_accessor :result, :mutex, :seen
 
+  RBBT_DEBUG_CLEAN = ENV["RBBT_DEBUG_CLEAN"] == 'true'
+
   class << self
     attr_accessor :lock_dir
     
@@ -454,6 +456,7 @@ class Step
     status << "not running" if ! done? && ! running? 
     status.unshift " " if status.any?
     Log.high "Cleaning step: #{path}#{status * " "}"
+    Log.stack caller if RBBT_DEBUG_CLEAN
     abort if ! done? && running?
     Step.clean(path)
     self

data/lib/rbbt/workflow/step/accessor.rb

@@ -8,6 +8,16 @@ class Step
                        end
                      end
 
+  def self.serialize_info(info)
+    info = info.clean_version if IndiferentHash === info
+    INFO_SERIALIZER.dump(info)
+  end
+
+  def self.load_serialized_info(io)
+    IndiferentHash.setup(INFO_SERIALIZER.load(io))
+  end
+
+
   def self.wait_for_jobs(jobs)
     jobs = [jobs] if Step === jobs
     begin
@@ -59,7 +69,7 @@ class Step
   def self.step_info(path)
     begin
       Open.open(info_file(path), :mode => 'rb') do |f|
-        INFO_SERIALIZER.load(f)
+        self.load_serialized_info(f)
       end
     rescue Exception
       Log.exception $!
@@ -188,7 +198,7 @@ class Step
                 info_lock.lock if check_lock and false
                 begin
                   Open.open(info_file, :mode => 'rb') do |file|
-                    IndiferentHash.setup(INFO_SERIALIZER.load(file)) #|| {}
+                    Step.load_serialized_info(file)
                   end
                 ensure
                   info_lock.unlock if check_lock and false
@@ -204,7 +214,7 @@ class Step
       Log.debug{"Error loading info file: " + info_file}
       Log.exception $!
       Open.rm info_file
-      Misc.sensiblewrite(info_file, INFO_SERIALIZER.dump({:status => :error, :messages => ["Info file lost"]}))
+      Misc.sensiblewrite(info_file, Step.serialize_info({:status => :error, :messages => ["Info file lost"]}))
       raise $!
     end
   end
@@ -214,8 +224,8 @@ class Step
     Open.lock(info_file, :lock => info_lock) do
       i = {:status => :waiting, :pid => Process.pid, :path => path}
       i[:dependencies] = dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]} if dependencies
-      @info_cache = i
-      Misc.sensiblewrite(info_file, INFO_SERIALIZER.dump(i), :force => true, :lock => false)
+      Misc.sensiblewrite(info_file, Step.serialize_info(i), :force => true, :lock => false)
+      @info_cache = IndiferentHash.setup(i)
       @info_cache_time = Time.now
     end
   end
@@ -227,9 +237,9 @@ class Step
     Open.lock(info_file, :lock => info_lock) do
       i = info(false).dup
       i[key] = value 
+      dump = Step.serialize_info(i)
       @info_cache = IndiferentHash.setup(i)
-      dump = INFO_SERIALIZER.dump(i)
-      Misc.sensiblewrite(info_file, dump, :force => true, :lock => false)
+      Misc.sensiblewrite(info_file, dump, :force => true, :lock => false) if Open.exists?(info_file)
       @info_cache_time = Time.now
       value
     end
@@ -242,9 +252,9 @@ class Step
     Open.lock(info_file, :lock => info_lock) do
       i = info(false)
       i.merge! hash
+      dump = Step.serialize_info(i)
       @info_cache = IndiferentHash.setup(i)
-      dump = INFO_SERIALIZER.dump(i)
-      Misc.sensiblewrite(info_file, dump, :force => true, :lock => false)
+      Misc.sensiblewrite(info_file, dump, :force => true, :lock => false) if Open.exists?(info_file)
       @info_cache_time = Time.now
       value
     end

data/lib/rbbt/workflow/step/dependencies.rb

@@ -92,7 +92,13 @@ class Step
           (job.done? && job.dirty?)  || (job.error? && job.dirty?) ||
           (!(job.noinfo? || job.done? || job.error? || job.aborted? || job.running?))
 
-        job.clean unless job.resumable? && (job.updated? && ! job.dirty?)
+        if ! (job.resumable? && (job.updated? && ! job.dirty?))
+          Log.high "About to clean -- status: #{status}, present #{File.exists?(job.path)}, " +
+            %w(done? error? recoverable_error? noinfo? updated? dirty? aborted? running? resumable?).
+            collect{|v| [v, job.send(v)]*": "} * ", " if RBBT_DEBUG_CLEAN
+
+          job.clean
+        end
         job.set_info :status, :cleaned
       end
 
@@ -121,7 +127,7 @@ class Step
   end
 
   def input_dependencies
-    inputs.flatten.select{|i| Step === i}
+    (inputs.flatten.select{|i| Step === i} + inputs.flatten.select{|dep| Path === dep && Step === dep.resource}.collect{|dep| dep.resource})
   end
 
 

data/lib/rbbt/workflow/step/run.rb

@@ -112,7 +112,7 @@ class Step
   end
 
   def updatable?
-    (ENV["RBBT_UPDATE_ALL_JOBS"] == 'true' || ( ENV["RBBT_UPDATE"] == "true" && Open.exists?(info_file)) && status != :noinfo && ! (relocated? && done?))
+    (ENV["RBBT_UPDATE_ALL_JOBS"] == 'true' || ( ENV["RBBT_UPDATE"] == "true" && Open.exists?(info_file)) && status != :noinfo && ! (relocated? && done?)) || (ENV["RBBT_UPDATE"] && ! (done? && ! Open.exists?(info_file)))
   end
 
   def dependency_checks
@@ -128,7 +128,7 @@ class Step
   end
 
   def input_checks
-    inputs.select{|i| Step === i }.
+    (inputs.select{|i| Step === i } + inputs.select{|i| Path === i && Step === i.resource}.collect{|i| i.resource}).
       select{|dependency| dependency.updatable? }
   end
 
@@ -154,25 +154,28 @@ class Step
     canfail_paths = self.canfail_paths
     this_mtime = Open.mtime(self.path) if Open.exists?(self.path)
 
-    checks.each do |dep| 
-      next unless dep.updatable?
-      dep_done = dep.done?
+    outdated_time = checks.select{|dep| dep.updatable? && dep.done? && Persist.newer?(path, dep.path) }
+    outdated_dep = checks.reject{|dep| dep.done? || (dep.error? && ! dep.recoverable_error? && canfail_paths.include?(dep.path)) }
 
-      begin
-        if this_mtime && dep_done && Open.exists?(dep.path) && (Open.mtime(dep.path) > this_mtime + 1)
-          outdated_time << dep
-        end
-      rescue
-      end
+    #checks.each do |dep| 
+    #  next unless dep.updatable?
+    #  dep_done = dep.done?
 
-      # Is this pointless? this would mean some dep got updated after a later
-      # dep but but before this one.
-      #if (! dep.done? && ! canfail_paths.include?(dep.path)) || ! dep.updated?
+    #  begin
+    #    if this_mtime && dep_done && Open.exists?(dep.path) && (Open.mtime(dep.path) > this_mtime + 1)
+    #      outdated_time << dep
+    #    end
+    #  rescue
+    #  end
 
-      if (! dep_done && ! canfail_paths.include?(dep.path))
-        outdated_dep << dep
-      end
-    end
+    #  # Is this pointless? this would mean some dep got updated after a later
+    #  # dep but but before this one.
+    #  #if (! dep.done? && ! canfail_paths.include?(dep.path)) || ! dep.updated?
+
+    #  if (! dep_done && ! canfail_paths.include?(dep.path))
+    #    outdated_dep << dep
+    #  end
+    #end
 
     Log.high "Some newer files found: #{Misc.fingerprint outdated_time}" if outdated_time.any?
     Log.high "Some outdated files found: #{Misc.fingerprint outdated_dep}" if outdated_dep.any?
@@ -215,7 +218,7 @@ class Step
         no_load = :stream if no_load
 
         Open.write(pid_file, Process.pid.to_s) unless Open.exists?(path) or Open.exists?(pid_file)
-        result_type = @task.result_type
+        result_type = @task.result_type if @task
         result_type = info[:result_type] if result_type.nil?
         result = Persist.persist "Job", result_type, :file => path, :check => persist_checks, :no_load => no_load do 
           if Step === Step.log_relay_step and not self == Step.log_relay_step

data/share/rbbt_commands/workflow/info

@@ -17,6 +17,7 @@ $ rbbt workflow info <job-result>
 -a--all Print all info entries
 -r--recursive Print recursive input values
 -o--original Print original object
+-w--width* Screen width
 EOF
 
 SOPT.usage if options[:help]
@@ -24,6 +25,7 @@ SOPT.usage if options[:help]
 file = ARGV.shift
 all = options.delete :all
 recursive = options.delete :recursive
+width = (options.delete(:width) || 80).to_i
 
 def get_step(file)
   file = file.sub(/\.(info|files)/,'')
@@ -86,6 +88,7 @@ pid = info[:pid]
 exception = info[:exception]
 rest = info.keys - [:inputs, :dependencies, :status, :time_elapsed, :messages, :backtrace, :exception, :pid, :archived_info]
 
+
 puts Log.color(:magenta, "File") << ": " << step.path
 puts Log.color(:magenta, "Status") << ": " << status_msg(status) << ((step.aborted? || step.error?) && step.recoverable_error? ? " (recoverable)" : "" ) << (step.dirty? ? " (dirty)" : "")
 puts Log.color(:magenta, "Pid") << ": " << pid_msg(pid, status.to_s == "done")
@@ -96,14 +99,14 @@ if inputs and inputs.any?
   inputs.each do |input,value|
     case value
     when nil
-      puts Misc.format_definition_list_item("  " + input.to_s, 'nil', 80, 20, :blue)
+      puts Misc.format_definition_list_item("  " + input.to_s, 'nil', width, 20, :blue)
     when Array
-      puts Misc.format_definition_list_item("  " + input.to_s, (value.length > 6 ? value[0..5]*"\n" << "\n" << "..." : value * "\n" ), 80, 20, :blue)
+      puts Misc.format_definition_list_item("  " + input.to_s, (value.length > 6 ? value[0..5]*"\n" << "\n" << "..." : value * "\n" ), width, 20, :blue)
     when TrueClass, FalseClass
-      puts Misc.format_definition_list_item("  " + input.to_s, value.to_s, 80, 20, :blue)
+      puts Misc.format_definition_list_item("  " + input.to_s, value.to_s, width, 20, :blue)
     else
       text = value.to_s.split("\n")[0..5].compact * "\n\n"
-      puts Misc.format_definition_list_item("  " + input.to_s, text, 80, 20, :blue)
+      puts Misc.format_definition_list_item("  " + input.to_s, text, width, 20, :blue)
     end
   end
 end
@@ -162,16 +165,16 @@ if recursive
     inputs.each do |input,value|
       case value
       when nil
-        puts Misc.format_definition_list_item("  " << input.to_s, 'nil', 80, 20, :blue)
+        puts Misc.format_definition_list_item("  " << input.to_s, 'nil', width, 20, :blue)
       when Array
-        puts Misc.format_definition_list_item("  " << input.to_s, (value.length > 6 ? (value[0..5])*"\n\n" << "\n\n" << "..." : value * "\n\n" ), 80, 20, :blue).gsub("\n\n","\n")
+        puts Misc.format_definition_list_item("  " << input.to_s, (value.length > 6 ? (value[0..5])*"\n\n" << "\n\n" << "..." : value * "\n\n" ), width, 20, :blue).gsub("\n\n","\n")
       when TrueClass, FalseClass
-        puts Misc.format_definition_list_item("  " << input.to_s, value.to_s, 80, 20, :blue)
+        puts Misc.format_definition_list_item("  " << input.to_s, value.to_s, width, 20, :blue)
       else
-        lines = value.to_s.split("\n").collect{|l| l.length >= 60 ? l[0..45] + " ..." : l }
+        lines = value.to_s.split("\n").collect{|l| l.length >= width - 5 ? l[0..width - 5] + " ..." : l }
         text = lines[0..5].compact * "\n\n"
         text << "\n\n...\n\n" if lines.length > 6
-        puts Misc.format_definition_list_item("  " << input.to_s, text, 80, 20, :blue).gsub("\n\n","\n")
+        puts Misc.format_definition_list_item("  " << input.to_s, text, width, 20, :blue).gsub("\n\n","\n")
       end
     end
   end

data/test/rbbt/test_workflow.rb

@@ -156,6 +156,15 @@ for this dependency
     Open.read(file).reverse
   end
 
+  task :create_file => :text do |file|
+    Open.write(file('a'), "A")
+    Open.write(file('b'), "B")
+    "DONE"
+  end
+
+  dep_task :reverse_step_file, TestWF, :reverse_file do |jobname, options, dependencies|
+    dep = dependencies.flatten.first
+  end
 
 end
 
@@ -440,4 +449,27 @@ class TestWorkflow < Test::Unit::TestCase
       end
     end
   end
+
+  def test_input_step_file_check
+    job = TestWF.job(:t3).recursive_clean
+    job.run
+    Misc.with_env "RBBT_UPDATE", 'true' do
+      assert job.checks.select{|d| d.task_name.to_s == "t1" }.any?
+      job = TestWF.job(:t3)
+      job.step(:t1).clean
+      assert job.checks.select{|d| d.task_name.to_s == "t1" }.empty?
+      job = TestWF.job(:t3).recursive_clean
+      job.run
+      assert job.checks.select{|d| d.task_name.to_s == "t1" }.any?
+      job = TestWF.job(:t3)
+      sleep 1
+      Open.touch job.step(:t1).path
+      Misc.with_env "RBBT_UPDATE", "false" do
+        assert job.updated?
+      end
+      Misc.with_env "RBBT_UPDATE", "true" do
+        assert ! job.updated?
+      end
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.28.11
+  version: 5.28.12
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-11-25 00:00:00.000000000 Z
+date: 2020-12-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake