rbbt-util 5.32.17 → 5.32.22

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2e7aa425415a69c6615edd3fd7f38ce4c0bba0b1a911e7f97e927bf433483401
-  data.tar.gz: d2e8045de7f45d961afd8671b694c4c92f629dbc41d96057c84b9e97fc8cf15c
+  metadata.gz: bff9b30f50c873f7db46003571a8557c0ea756aaf342041119d4737da78e461b
+  data.tar.gz: 0765a797be1c16f90cd5ebf4f18fe5adb3d154aa69d07ab666cfe464fcb0a6bd
 SHA512:
-  metadata.gz: b4f577d5d647b6c9ce1226062e7f5a459e6dee13e151e782d5938494a3e061d4b3fcc41bd14a8276ea646e58b726571445a45db7f2fe03dc34f4ca28d94c2d9a
-  data.tar.gz: d8432ea0d5232ddd9ac89c9e9cf0880a677cf708772ea5d0a6b6e28bd2204831d676e613a218c616c950c509c381a0ab2cf1acf88b98a85e9d4f18c3480ef48c
+  metadata.gz: 9ba97c967d615145553d0f73bd84debbceee41c7ce1ca0c4fda512eeaf54880c256d36a4cf9d215ddce97b43a44a2cf19aa17f7e3b086512633210401cdfc085
+  data.tar.gz: c58f507abee878f56945886b06582271ace368c00808ff6fc7d0f0343373070a40dac797aff440a8abffd8cce3647fec9affeff078150bab8b7f52625754c91b

data/lib/rbbt/hpc/slurm.rb

@@ -44,6 +44,7 @@ export BATCH_SYSTEM=SLURM
       time = Misc.format_seconds Misc.timespan(time) unless time.include? ":"
 
       sbatch_params = {"job-name" => batch_name,
+                       "qos" => queue,
                        "output" => fout,
                        "error" => ferr,
                        "cpus-per-task" => task_cpus,

data/lib/rbbt/util/config.rb

@@ -122,7 +122,12 @@ module Rbbt::Config
     Log.debug "Value #{value.inspect} for config key '#{ key }': #{tokens * ", "}"
     GOT_KEYS << [key, value, tokens]
 
-    value
+    if String === value && m = value.match(/^env:(.*)/)
+      variable = m.captures.first
+      ENV[variable]
+    else
+      value
+    end
   end
 
   def self.with_config

data/lib/rbbt/util/misc/inspect.rb

@@ -320,7 +320,9 @@ module Misc
               end
             end
           when String
-            if Misc.is_filename?(obj, false) and ! %w(. ..).include?(obj) and ! Path === obj
+            good_filename = Misc.is_filename?(obj, false) && ! %w(. ..).include?(obj) && %w(. /).include?(obj[0])
+            is_path = Path === obj
+            if good_filename && ! is_path
               obj2str Path.setup(obj.dup)
             else
               obj = obj.chomp if String === obj

data/lib/rbbt/workflow/definition.rb

@@ -77,15 +77,15 @@ module Workflow
     returns workflow.tasks[oname].result_description if workflow.tasks.include?(oname) unless @result_description 
     task name do
       raise RbbtException, "dependency not found in dep_task" if dependencies.empty?
-      Step.wait_for_jobs dependencies
+      Step.wait_for_jobs dependencies.select{|d| d.streaming? }
       dep = dependencies.last
       dep.join
       raise dep.get_exception if dep.error?
       raise Aborted, "Aborted dependency #{dep.path}" if dep.aborted?
       set_info :result_type, dep.info[:result_type]
-      forget = config :forget_dep_tasks, "forget_dep_tasks", "key:forget_dep_tasks", :default => FORGET_DEP_TASKS
+      forget = config :forget_dep_tasks, "forget_dep_tasks", :default => FORGET_DEP_TASKS
       if forget
-        remove = config :remove_dep_tasks, "remove_dep_tasks", "key:remove_dep_tasks", :default => REMOVE_DEP_TASKS
+        remove = config :remove_dep_tasks, "remove_dep_tasks", :default => REMOVE_DEP_TASKS
 
         self.archive_deps
         self.copy_files_dir
@@ -93,15 +93,17 @@ module Workflow
         Open.rm_rf self.files_dir if Open.exist? self.files_dir
         FileUtils.cp_r dep.files_dir, self.files_dir if Open.exist?(dep.files_dir)
         Open.ln_h dep.path, self.tmp_path
+
         case remove.to_s
         when 'true'
           dep.clean
         when 'recursive'
-          dep.rec_dependencies.each do |d|
+          (dep.dependencies + dep.rec_dependencies).uniq.each do |d|
+            next if d.overriden
             d.clean unless config(:remove_dep, d.task_signature, d.task_name, d.workflow.to_s, :default => true).to_s == 'false'
           end
           dep.clean unless config(:remove_dep, dep.task_signature, dep.task_name, dep.workflow.to_s, :default => true).to_s == 'false'
-        end
+        end unless dep.overriden
       else
         if Open.exists?(dep.files_dir)
           Open.rm_rf self.files_dir 

data/lib/rbbt/workflow/dependencies.rb

@@ -71,7 +71,7 @@ module Workflow
 
   def setup_override_dependency(dep, workflow, task_name)
     return [] if dep == :skip || dep == 'skip'
-    dep = Step === dep ? dep : Workflow.load_step(dep)
+    dep = Step === dep ? dep.dup : Workflow.load_step(dep)
     dep.workflow = workflow
     dep.info[:name] = dep.name
     dep.original_task_name ||= dep.task_name if dep.workflow
@@ -83,8 +83,9 @@ module Workflow
     rescue
       Log.exception $!
     end
+
     dep.task_name = task_name
-    dep.overriden = dep.original_task_name.to_sym
+    dep.overriden = dep.original_task_name.to_sym if dep.original_task_name
 
     dep.extend step_module
 

data/lib/rbbt/workflow/step.rb

@@ -351,38 +351,6 @@ class Step
     end
   end
 
-  #connected = true means that dependency searching ends when a result is done
-  #but dependencies are absent, meanining that the file could have been dropped
-  #in
-  def rec_dependencies(connected = false, seen = [])
-
-    # A step result with no info_file means that it was manually
-    # placed. In that case, do not consider its dependencies
-    return [] if ! (defined? WorkflowRemoteClient && WorkflowRemoteClient::RemoteStep === self) && ! Open.exists?(self.info_file) && Open.exists?(self.path.to_s) 
-
-    return [] if dependencies.nil? or dependencies.empty?
-
-    new_dependencies = []
-    if self.overriden?
-      archived_deps = []
-    else
-      archived_deps = self.info[:archived_info] ? self.info[:archived_info].keys : []
-    end
-
-    dependencies.each{|step| 
-      #next if self.done? && Open.exists?(info_file) && info[:dependencies] && info[:dependencies].select{|task,name,path| path == step.path }.empty?
-      next if archived_deps.include? step.path
-      next if seen.include? step.path
-      next if self.done? && connected && ! updatable?
-
-      r = step.rec_dependencies(connected, new_dependencies.collect{|d| d.path})
-      new_dependencies.concat r
-      new_dependencies << step
-    }
-
-    new_dependencies.uniq
-  end
-
   def writable?
     Open.writable?(self.path) && Open.writable?(self.info_file)
   end

data/lib/rbbt/workflow/step/dependencies.rb

@@ -206,7 +206,7 @@ class Step
     if dup and step.streaming? and not step.result.nil?
       if dep_step[step.path] and dep_step[step.path].length > 1
         stream = step.result
-        other_steps = dep_step[step.path].uniq
+        other_steps = dep_step[step.path].uniq.reject{|d| d.overriden }
         return unless other_steps.length > 1
         log_dependency_exec(step, "duplicating #{other_steps.length}") 
         copies = Misc.tee_stream_thread_multiple(stream, other_steps.length)
@@ -523,4 +523,36 @@ class Step
     set_info :dependencies, dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]} if dependencies
   end
 
+  #connected = true means that dependency searching ends when a result is done
+  #but dependencies are absent, meanining that the file could have been dropped
+  #in
+  def rec_dependencies(connected = false, seen = [])
+
+    # A step result with no info_file means that it was manually
+    # placed. In that case, do not consider its dependencies
+    return [] if ! (defined? WorkflowRemoteClient && WorkflowRemoteClient::RemoteStep === self) && ! Open.exists?(self.info_file) && Open.exists?(self.path.to_s) 
+
+    return [] if dependencies.nil? or dependencies.empty?
+
+    new_dependencies = []
+    if self.overriden?
+      archived_deps = []
+    else
+      archived_deps = self.info[:archived_info] ? self.info[:archived_info].keys : []
+    end
+
+    dependencies.each{|step| 
+      #next if self.done? && Open.exists?(info_file) && info[:dependencies] && info[:dependencies].select{|task,name,path| path == step.path }.empty?
+      next if archived_deps.include? step.path
+      next if seen.include? step.path
+      next if self.done? && connected && ! updatable?
+
+      r = step.rec_dependencies(connected, new_dependencies.collect{|d| d.path})
+      new_dependencies.concat r
+      new_dependencies << step
+    }
+
+    new_dependencies.uniq
+  end
+
 end

data/lib/rbbt/workflow/util/provenance.rb

@@ -102,7 +102,7 @@ class Step
         dep.input_dependencies.each do |id|
           input_name = dep.recursive_inputs.fields.zip(dep.recursive_inputs).select{|f,d| 
             d == id || (String === d && d.start_with?(id.files_dir)) || (Array === d && d.include?(id))
-          }.first.first
+          }.last.first
           input_dependencies[id] ||= []
           input_dependencies[id] << [dep, input_name]
         end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.32.17
+  version: 5.32.22
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-07-13 00:00:00.000000000 Z
+date: 2021-07-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake