rbbt-util 5.34.26 → 5.34.27

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: da3b496fd0fb7d9ff645cba0fb4ba3914288fb0b88bf80f43a79d02bd1ee2cfd
-  data.tar.gz: ab120f9bb96f1b0628f1edc6dc114a43c32ed39a5c29ce5252bbf85383d24359
+  metadata.gz: 40a3d86f01180255239e05fc23309b2c798eb94ee4c213ab8699f7954c1f5346
+  data.tar.gz: 3357e23002f032213a6b4c01ef046527092c8e4bf9ebce96dfc714176c3cb239
 SHA512:
-  metadata.gz: 3f5b5326a834f1f972756919ea2f04c9626998be3c9ca058fd386c06787011ad021798e6091eb8eae1f7873dcceb9adcfa02de14a5269dadef4a2902d5475a9d
-  data.tar.gz: d74344affb899a6b62a494d02e9058dc338a7be4138b9ce32331d0657bc121035b312da21f48827751dc13e1bdee4511fb6f1eeaf481d39f73e5967ff3243800
+  metadata.gz: 8454ec0e14dac9fbdb5fa00a2219248b5037c890289fb5b013d69d6dac63e869b5160620d1ac744392546bf58dc8acfb1afba5c9172f366ddae01fc6cc008810
+  data.tar.gz: c85e5b6defde582d6af2c08281dd659bd8f71ae9867d4258d2c96c9824b3d0fe7020472fc6ccec3e66f4f70228bdc3dea425756abd510ded61a7f59420b78b98

data/lib/rbbt/hpc/batch.rb

@@ -560,6 +560,16 @@ env > #{batch_options[:fenv]}
     def run_job(job, options = {})
       system = self.to_s.split("::").last
 
+      if (batch_job = job.info[:batch_job]) && job_queued(batch_job)
+        Log.info "Job #{job.short_path} already queued in #{batch_job}"
+        return batch_job 
+      end
+
+      if job.running?
+        Log.info "Job #{job.short_path} already running in #{job.info[:pid]}"
+        return job.info[:batch_job]
+      end
+
       batch_base_dir, clean_batch_job, remove_batch_dir, procpath, tail, batch_dependencies, dry_run = Misc.process_options options, 
         :batch_base_dir, :clean_batch_job, :remove_batch_dir, :batch_procpath, :tail, :batch_dependencies, :dry_run,
         :batch_base_dir => File.expand_path(File.join('~/rbbt-batch')) 
@@ -583,6 +593,8 @@ env > #{batch_options[:fenv]}
 
         batch_job = run_template(batch_dir, dry_run)
 
+        hold_dependencies(job, batch_job) unless dry_run
+
         return [batch_job, batch_dir] unless tail
 
         t_monitor = Thread.new do
@@ -603,6 +615,17 @@ env > #{batch_options[:fenv]}
       end
     end
 
+    def hold_dependencies(job, batch_job)
+      job.set_info :batch_job, batch_job
+      job.set_info :batch_system, self.batch_system
+      job.dependencies.each do |dep|
+        next unless dep.waiting?
+        next if (dep_batch_job = dep.info[:batch_job]) && job_queued(dep_batch_job)
+
+        hold_dependencies(dep, batch_job)
+      end
+    end
+
     def follow_job(batch_dir, tail = true)
       fjob = File.join(batch_dir, 'job.id')
       fout = File.join(batch_dir, 'std.out')
@@ -672,7 +695,7 @@ env > #{batch_options[:fenv]}
           terr = Misc.consume_stream(err, true, STDERR) if err
           tout = Misc.consume_stream(out, true, STDOUT) if out
 
-          sleep 3 while job_status(job).include? job.to_s
+          sleep 3 while job_queued(job)
         rescue Aborted
         ensure
           begin
@@ -692,6 +715,10 @@ env > #{batch_options[:fenv]}
       end
     end
 
+    def job_queued(job)
+      job_status(job).split(/\s+/).include?(job.to_s)
+    end
+
     def wait_for_job(batch_dir, time = 1)
       fexit = File.join(batch_dir, 'exit.status')
       fjob = File.join(batch_dir, 'job.id')

data/lib/rbbt/hpc/lsf.rb

@@ -5,6 +5,10 @@ module HPC
     extend HPC::TemplateGeneration
     extend HPC::Orchestration
 
+    def self.batch_system
+      "LSF"
+    end
+
     def self.batch_system_variables
       <<-EOF
 let TOTAL_PROCESORS="$(cat /proc/cpuinfo|grep ^processor |wc -l)"
@@ -13,7 +17,7 @@ let MAX_MEMORY_DEFAULT="$(grep MemTotal /proc/meminfo|grep -o "[[:digit:]]*") /
 export MAX_MEMORY_DEFAULT
 export MAX_MEMORY
 export BATCH_JOB_ID=$LSF_JOBID
-export BATCH_SYSTEM=LSF
+export BATCH_SYSTEM=#{batch_system}
       EOF
     end
 

data/lib/rbbt/hpc/orchestrate.old.rb

@@ -206,7 +206,7 @@ module HPC
       options.delete "detach"
       options.delete "jobname"
 
-      rules = YAML.load(Open.read(options[:orchestration_rules])) if options[:orchestration_rules]
+      rules = Misc.load_yaml(options[:orchestration_rules]) if options[:orchestration_rules]
       rules ||= {}
       IndiferentHash.setup(rules)
 

data/lib/rbbt/hpc/orchestrate.rb

@@ -37,9 +37,9 @@ module HPC
       prepare_for_execution(job)
 
       if options[:orchestration_rules]
-        rules = YAML.load(Open.read(options[:orchestration_rules]))
+        rules = Misc.load_yaml(options[:orchestration_rules])
       elsif Rbbt.etc.slurm["default.yaml"].exists?
-        rules = YAML.load(Open.read(Rbbt.etc.slurm["default.yaml"]))
+        rules = Misc.load_yaml(Rbbt.etc.slurm["default.yaml"])
       else
         rules = {}
       end

data/lib/rbbt/hpc/slurm.rb

@@ -6,6 +6,10 @@ module HPC
     extend HPC::TemplateGeneration
     extend HPC::Orchestration
 
+    def self.batch_system
+      "SLURM"
+    end
+
     def self.batch_system_variables
       <<-EOF
 let TOTAL_PROCESORS="$(cat /proc/cpuinfo|grep ^processor |wc -l)"
@@ -16,7 +20,7 @@ MAX_MEMORY="$MAX_MEMORY_DEFAULT"
 export MAX_MEMORY_DEFAULT
 export MAX_MEMORY
 export BATCH_JOB_ID=$SLURM_JOB_ID
-export BATCH_SYSTEM=SLURM
+export BATCH_SYSTEM=#{batch_system}
       EOF
     end
 
@@ -145,7 +149,11 @@ export BATCH_SYSTEM=SLURM
       if job.nil?
         CMD.cmd("squeue").read
       else
-        CMD.cmd("squeue --job #{job}").read
+        begin
+          CMD.cmd("squeue --job #{job}").read
+        rescue
+          ""
+        end
       end
     end
 

data/lib/rbbt/monitor.rb

@@ -64,14 +64,14 @@ module Rbbt
         lock_info[f].merge!(file_time(f))
         if File.size(f) > 0
           info = Open.open(f) do |s|
-            YAML.load(s)
+            Misc.load_yaml(s)
           end
           IndiferentHash.setup(info)
           lock_info[f][:pid] = info[:pid]
           lock_info[f][:ppid] = info[:ppid]
         end
       rescue Exception
-        #Log.exception $!
+        Log.exception $!
       end
     end
     lock_info
@@ -272,7 +272,7 @@ module Rbbt
   def self.load_lock(lock)
     begin
       info = Misc.insist 3 do
-        YAML.load(Open.read(lock))
+        Misc.load_yaml(lock)
       end
       info.values_at "pid", "ppid", "time"
     rescue Exception

data/lib/rbbt/persist.rb

@@ -148,9 +148,7 @@ module Persist
           JSON.parse(stream.read)
         end
       when :yaml
-        Open.open(path) do |stream|
-          YAML.load(stream)
-        end
+        Misc.load_yaml(path)
       when :float
         Open.read(path).to_f
       when :integer

data/lib/rbbt/resource/path.rb

@@ -150,7 +150,7 @@ module Path
   search_path_file = File.join(ENV['HOME'], '.rbbt/etc/search_paths')
   if File.exist?(search_path_file)
     begin
-      YAML.load(File.open(search_path_file)).each do |where, location|
+      Misc.load_yaml(search_path_file).each do |where, location|
         SEARCH_PATHS[where.to_sym] = location
       end
     rescue

data/lib/rbbt/util/misc/serialize.rb

@@ -0,0 +1,26 @@
+module Misc
+  def self.load_yaml(yaml)
+    case yaml
+    when IO, StringIO
+      if YAML.respond_to?(:unsafe_load)
+        YAML.unsafe_load(yaml)
+      else
+        YAML.load(yaml)
+      end
+    when (defined?(Path) && Path)
+      yaml.open do |io|
+        load_yaml(io)
+      end
+    when String
+      if Misc.is_filename?(yaml)
+        File.open(yaml) do |io|
+          load_yaml(io)
+        end
+      else
+        load_yaml(StringIO.new(yaml))
+      end
+    else
+      raise "Unkown YAML object: #{Misc.fingerprint yaml}"
+    end
+  end
+end

data/lib/rbbt/util/misc.rb

@@ -22,6 +22,8 @@ require 'rbbt/util/misc/objects'
 require 'rbbt/util/misc/manipulation'
 require 'rbbt/util/misc/communication'
 
+require 'rbbt/util/misc/serialize'
+
 require 'to_regexp'
 
 module MultipleResult; end

data/lib/rbbt/workflow/step/accessor.rb

@@ -143,8 +143,12 @@ class Step
 
   def init_info(force = false)
     return nil if @exec || info_file.nil? || (Open.exists?(info_file) && ! force)
+    batch_job = info[:batch_job] if Open.exists?(info_file)
+    batch_system = info[:batch_system] if Open.exists?(info_file)
     Open.lock(info_file, :lock => info_lock) do
       i = {:status => :waiting, :pid => Process.pid, :path => path, :real_inputs => real_inputs, :overriden => overriden}
+      i[:batch_job] = batch_job if batch_job
+      i[:batch_system] = batch_system if batch_system
       i[:dependencies] = dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]} if dependencies
       Misc.sensiblewrite(info_file, Step.serialize_info(i), :force => true, :lock => false)
       @info_cache = IndiferentHash.setup(i)
@@ -253,7 +257,7 @@ class Step
       #Open.read(file(name)).split /\n|,\s*/
       Open.read(file(name)).split "\n"
     when :yaml
-      YAML.load(Open.open(file(name)))
+      Misc.load_yaml(file(name))
     when :marshal
       Marshal.load(Open.open(file(name)))
     else

data/lib/rbbt/workflow/step/run.rb

@@ -698,7 +698,7 @@ class Step
   end
 
   def grace
-    until done? || result || error? || aborted? || streaming? || waiting? 
+    until done? || result || error? || aborted? || streaming? || waiting? || running?
       sleep 1 
     end
     self

data/lib/rbbt/workflow/step/save_load_inputs.rb

@@ -82,7 +82,7 @@ module Workflow
         when :file, :binary
           Log.debug "Pointing #{ input } to #{file}"
           if file =~ /\.yaml/
-            inputs[input.to_sym]  = YAML.load(Open.read(file))
+            inputs[input.to_sym]  = Misc.load_yaml(file)
           else
             if File.symlink?(file)
               link_target = File.expand_path(File.readlink(file), File.dirname(file))

data/lib/rbbt/workflow/step/status.rb

@@ -15,7 +15,7 @@ class Step
       raise "DO NOT CLEAN" 
     end
 
-    if (Open.exists?(path) or Open.broken_link?(path)) or Open.exists?(pid_file) or Open.exists?(info_file) or Open.exists?(files_dir) or Open.broken_link?(files_dir)
+    if (Open.exists?(path) or Open.broken_link?(path)) or Open.exists?(pid_file) or Open.exists?(info_file) or Open.exists?(files_dir) or Open.broken_link?(files_dir) or Open.exists?(pid_file)
 
       @result = nil
       @pid = nil

data/lib/rbbt/workflow.rb

@@ -834,8 +834,7 @@ module Workflow
   end
 
   def self.load_remote_tasks(filename)
-    yaml_text = Open.read(filename)
-    remote_workflow_tasks = YAML.load(yaml_text)
+    remote_workflow_tasks = Misc.load_yaml(filename)
     Workflow.process_remote_tasks(remote_workflow_tasks)
   end
 
@@ -859,8 +858,7 @@ module Workflow
   end
 
   def self.load_relay_tasks(filename)
-    yaml_text = Open.read(filename)
-    relay_workflow_tasks = YAML.load(yaml_text)
+    remote_workflow_tasks = Misc.load_yaml(filename)
     Workflow.process_relay_tasks(relay_workflow_tasks)
   end
 end

data/share/rbbt_commands/system/clean

@@ -46,7 +46,7 @@ if locks.any?
   puts
   puts Log.color(:magenta, "Locks:")
   locks.each do |file,info|
-    if force or (info[:pid] and not Misc.pid_exists? info[:pid])
+    if force or (info[:pid] && ! Misc.pid_exists?(info[:pid]))
       puts "  Removing #{ file }"
       File.unlink file 
     end

data/test/rbbt/util/misc/test_serialize.rb

@@ -0,0 +1,24 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
+require 'rbbt/util/misc/serialize'
+
+class TestClass < Test::Unit::TestCase
+  def test_load_yaml
+    yaml_txt=<<-EOF
+---
+a: b
+    EOF
+    yaml_sio = StringIO.new yaml_txt
+
+    assert_equal "b", Misc.load_yaml(yaml_txt)["a"]
+    assert_equal "b", Misc.load_yaml(yaml_sio)["a"]
+
+    TmpFile.with_file yaml_txt do |yaml_file|
+      assert_equal "b", Misc.load_yaml(yaml_file)["a"]
+      Open.open(yaml_file) do |yaml_io|
+        assert_equal "b", Misc.load_yaml(yaml_io)["a"]
+      end
+    end
+
+  end
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.34.26
+  version: 5.34.27
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-12-14 00:00:00.000000000 Z
+date: 2022-12-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -323,6 +323,7 @@ files:
 - lib/rbbt/util/misc/omics.rb
 - lib/rbbt/util/misc/options.rb
 - lib/rbbt/util/misc/pipes.rb
+- lib/rbbt/util/misc/serialize.rb
 - lib/rbbt/util/misc/system.rb
 - lib/rbbt/util/named_array.rb
 - lib/rbbt/util/open.rb
@@ -559,6 +560,7 @@ files:
 - test/rbbt/util/misc/test_multipart_payload.rb
 - test/rbbt/util/misc/test_omics.rb
 - test/rbbt/util/misc/test_pipes.rb
+- test/rbbt/util/misc/test_serialize.rb
 - test/rbbt/util/python/test_util.rb
 - test/rbbt/util/simpleopt/test_get.rb
 - test/rbbt/util/simpleopt/test_parse.rb
@@ -634,6 +636,7 @@ test_files:
 - test/rbbt/util/misc/test_development.rb
 - test/rbbt/util/misc/test_omics.rb
 - test/rbbt/util/misc/test_pipes.rb
+- test/rbbt/util/misc/test_serialize.rb
 - test/rbbt/util/misc/test_format.rb
 - test/rbbt/util/misc/test_communication.rb
 - test/rbbt/util/misc/test_lock.rb