rbbt-util 5.25.38 → 5.25.39

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0d1025ab83e8ce62b9161450d75182943daccf99
-  data.tar.gz: adaacf924bff1a24e93056682c2079def611afc7
+  metadata.gz: 0f56f0578a67dbc452f235f6ccc30c8c9c97dbfb
+  data.tar.gz: 0c333ea486edd8afead278153fec7a4c81e2a12e
 SHA512:
-  metadata.gz: c4b6bd64837133aec8f7333d7179a21b3344adb9b84dd201e224487eb3236d34b20215851ec7dab00adefe049588a47ef1fd8c2a839c6dcecf742436508948d3
-  data.tar.gz: e9eee81ad6f882d4234214f10c41438f2a63bc1df9971b75d1b2f87ee37751e81f3f57bece1cf66d199080d9853530a12b404c6e0f5ff19ab3567844d1adc45d
+  metadata.gz: 4839d7df1d9ac6bdaadaad1dca2b4f57d3aefedb55323f1edccb8de3d9fedc49c44eecd26db5ea8d2469c8fe073b34bb513f37a62ad7268ac2dfcd536a2524a5
+  data.tar.gz: 3bde4b44aeceac13e2129f5b236e72fe0c411f7f0b1f0d8cf339584030a64e64aa07b1d360a4f1707d5aa125dbce65d96ab6eb28d050397a00112dd6eb08042f

data/bin/rbbt

@@ -98,15 +98,19 @@ if mem_dump = options.delete(:dump_mem)
 end
 
 if options[:config_keys]
-  options[:config_keys].split(",").each do |config|
-    key, value, *tokens = config.split(/\s/)
-    tokens = ['key:' << key << '::0'] if tokens.empty?
-    tokens = tokens.collect do |tok|
-      tok, _sep, prio = tok.partition("::")
-      prio = "0" if prio.nil? or prio.empty?
-      [tok, prio] * "::"
+  if Misc.is_filename?(options[:config_keys]) && File.exists?(options[:config_keys])
+    Rbbt::Config.load_file(options[:config_keys])
+  else
+    options[:config_keys].split(",").each do |config|
+      key, value, *tokens = config.split(/\s/)
+      tokens = ['key:' << key << '::0'] if tokens.empty?
+      tokens = tokens.collect do |tok|
+        tok, _sep, prio = tok.partition("::")
+        prio = "0" if prio.nil? or prio.empty?
+        [tok, prio] * "::"
+      end
+      Rbbt::Config.set({key => value}, *tokens)
     end
-    Rbbt::Config.set({key => value}, *tokens)
   end
 end
 

data/lib/rbbt/hpc.rb

@@ -43,7 +43,7 @@ module Marenostrum
       rbbt_cmd << " --config_keys='#{config_keys}'"
 
       queue = options[:queue] || 'bsc_ls'
-      tasks = options[:tasks] || 1
+      task_cpus = options[:task_cpus] || 1
       nodes = options[:nodes] || 1
       time = options[:time] || "0:00:10"
 
@@ -71,7 +71,7 @@ module Marenostrum
 #SBATCH --workdir="#{Dir.pwd}"
 #SBATCH --output="#{fout}"
 #SBATCH --error="#{ferr}"
-#SBATCH --ntasks="#{tasks}"
+#SBATCH --cpus-per-task="#{task_cpus}"
 #SBATCH --time="#{time}"
 #SBATCH --nodes="#{nodes}"
       EOF

data/lib/rbbt/util/config.rb

@@ -5,6 +5,8 @@ module Rbbt::Config
 
   CACHE = IndiferentHash.setup({})
 
+  GOT_KEYS=[]
+
   def self.add_entry(key, value, tokens)
     CACHE[key.to_s] ||= [] 
     CACHE[key.to_s] << [tokens, value]
@@ -110,7 +112,10 @@ module Rbbt::Config
 
     value = priorities.empty? ? default : priorities.sort_by{|p,v| p}.first.last.first
     value = false if value == 'false'
+
     Log.debug "Value #{value.inspect} for config key '#{ key }': #{tokens * ", "}"
+    GOT_KEYS << [key, value, tokens]
+
     value
   end
 

data/lib/rbbt/util/simpleopt/get.rb

@@ -1,4 +1,5 @@
 module SOPT
+  GOT_OPTIONS= IndiferentHash.setup({})
   def self.consume(args = ARGV)
     i = 0
     values = {}
@@ -35,6 +36,7 @@ module SOPT
     end
 
     IndiferentHash.setup values
+    GOT_OPTIONS.merge! values
 
     values
   end

data/lib/rbbt/workflow/accessor.rb

@@ -747,7 +747,7 @@ module Workflow
                                   when Array
                                     wf, t, o = dep
 
-                                    wf.rec_dependencies(t).each do |d|
+                                    wf.rec_dependencies(t.to_sym).each do |d|
                                       if Array === d
                                         new = d.dup
                                       else

data/lib/rbbt/workflow/provenance.rb

@@ -0,0 +1,93 @@
+class Step
+  def self.prov_status_msg(status)
+    color = case status.to_sym
+            when :error, :aborted, :missing, :dead
+              :red
+            when :streaming, :started
+              :cyan
+            when :done
+              :green
+            when :noinfo
+              :blue
+            when :dependencies, :waiting, :setyp
+              :yellow
+            else
+              if status.to_s.index ">"
+                :cyan
+              else
+                :cyan
+              end
+            end
+    Log.color(color, status.to_s)
+  end
+
+  def self.prov_report_msg(status, name, path, info = nil)
+    parts = path.sub(/\{.*/,'').sub(/#{Regexp.quote(name)}$/,'').split "/"
+
+    task = Log.color(:yellow, parts.pop)
+    workflow = Log.color(:magenta, parts.pop)
+    if status.to_s == 'noinfo' and parts.last != 'jobs'
+      task, status, workflow = Log.color(:yellow, info[:task_name]), Log.color(:blue, "file"), Log.color(:magenta, "-")
+    end
+
+    path_mtime = begin
+                   Open.mtime(path)
+                 rescue
+                   nil
+                 end
+    str = if not Open.remote?(path) and (Open.exists?(path) and $main_mtime and path_mtime and ($main_mtime - path_mtime) < -2)
+            iii [path, path_mtime, $main_mtime, $main_mtime - path_mtime]
+            prov_status_msg(status.to_s) << " " << [workflow, task, path].compact * " " << " (#{Log.color(:red, "Mtime out of sync") })"
+          else
+            prov_status_msg(status.to_s) << " " << [workflow, task, path].compact * " " 
+          end
+
+    if $inputs and $inputs.any? 
+      job_inputs = Workflow.load_step(path).recursive_inputs.to_hash
+      IndiferentHash.setup(job_inputs)
+
+      $inputs.each do |input|
+        value = job_inputs[input]
+        next if  value.nil?
+        value_str = Misc.fingerprint(value)
+        str << "\t#{Log.color :magenta, input}=#{value_str}"
+      end
+    end
+
+    if $info_fields and $info_fields.any?
+      $info_fields.each do |field|
+        IndiferentHash.setup(info)
+        value = info[field]
+        next if value.nil?
+        value_str = Misc.fingerprint(value)
+        str << "\t#{Log.color :magenta, field}=#{value_str}"
+      end
+    end
+
+    str << "\n"
+  end
+
+  def self.prov_report(step, offset = 0, task = nil)
+    info = step.info  || {}
+    info[:task_name] = task
+    path  = step.path
+    status = info[:status] || :missing
+    status = "remote" if Open.remote?(path)
+    name = info[:name] || File.basename(path)
+    status = :unsync if status == :done and not Open.exist? path
+    str = " " * offset
+    str << prov_report_msg(status, name, path, info)
+    seen = []
+    step.dependencies.each do |dep|
+      path = dep.path
+      new = ! seen.include?(path)
+      if new
+        seen << path
+        str << prov_report(dep, offset + 1, task)
+      else
+        str << Log.color(:blue, Log.uncolor(prov_report(dep, offset+1, task)))
+      end
+    end if step.dependencies
+    str
+  end
+end

data/lib/rbbt/workflow/step/dependencies.rb

@@ -385,7 +385,14 @@ class Step
       required_dep_paths << path if list.length > 1
     end
 
-    required_dep_paths.concat dependencies.collect{|dep| dep.path }
+    required_dep_paths.concat dependencies.collect{|dep| dep.path}
+    required_dep_paths.concat(rec_dependencies.collect do |dep| 
+      dep.inputs.flatten.select{|i| Step === i}.collect{|d| d.path}
+    end.flatten)
+
+    required_dep_paths.concat(dependencies.collect do |dep| 
+      [dep.path] + dep.inputs.flatten.select{|i| Step === i}.collect{|d| d.path}
+    end.flatten)
 
     log :dependencies, "Dependencies for step #{Log.color :yellow, task.name.to_s || ""}"
 
@@ -398,7 +405,7 @@ class Step
       next if seen_paths.include? step.path
       seen_paths << step.path
       next unless required_dep_paths.include? step.path
-      if dependencies.include?(step) and step.inputs.flatten.select{|i| Step === i}.any?
+      if step.inputs.flatten.select{|i| Step === i}.any?
         if ComputeDependency === step
           next if produced.include? step.path 
           compute_last_deps[step.compute] ||= []
@@ -417,6 +424,11 @@ class Step
       end
     end
 
+    Log.medium "Computing pre dependencies: #{Misc.fingerprint(compute_pre_deps)} - #{Log.color :blue, self.path}" if compute_pre_deps.any?
+    compute_pre_deps.each do |type,list|
+      run_compute_dependencies(type, list, dep_step)
+    end
+
     Log.medium "Processing pre dependencies: #{Misc.fingerprint(pre_deps)} - #{Log.color :blue, self.path}" if pre_deps.any?
     pre_deps.each do |step|
       next if compute_deps.include? step
@@ -427,11 +439,6 @@ class Step
       end
     end
 
-    Log.medium "Computing pre dependencies: #{Misc.fingerprint(compute_pre_deps)} - #{Log.color :blue, self.path}" if compute_pre_deps.any?
-    compute_pre_deps.each do |type,list|
-      run_compute_dependencies(type, list, dep_step)
-    end
-
     Log.medium "Processing last dependencies: #{Misc.fingerprint(last_deps)} - #{Log.color :blue, self.path}" if last_deps.any?
     last_deps.each do |step|
       next if compute_deps.include? step

data/lib/rbbt/workflow/step/run.rb

@@ -219,6 +219,8 @@ class Step
 
           Open.write(pid_file, Process.pid.to_s) unless Open.exists? pid_file
 
+          config_keys_pre = Rbbt::Config::GOT_KEYS.dup
+
           @exec = false
           init_info(true)
 
@@ -405,6 +407,8 @@ class Step
 
           set_info :dependencies, dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]}
 
+          set_info :config_keys, (Rbbt::Config::GOT_KEYS - config_keys_pre).uniq
+
           if result.nil? && File.exists?(self.tmp_path) && ! File.exists?(self.path)
             Open.mv self.tmp_path, self.path
           end

data/share/rbbt_commands/workflow/prov

@@ -6,6 +6,7 @@ require 'rbbt-util'
 require 'fileutils'
 require 'rbbt/util/simpleopt'
 require 'rbbt/workflow/step'
+require 'rbbt/workflow/provenance'
 require 'rbbt/util/misc'
 
 require 'rbbt-util'
@@ -40,74 +41,6 @@ def get_step(file)
   Workflow.load_step file
 end
 
-def status_msg(status)
-  color = case status.to_sym
-          when :error, :aborted, :missing, :dead
-            :red
-          when :streaming, :started
-            :cyan
-          when :done
-            :green
-          when :noinfo
-            :blue
-          when :dependencies, :waiting, :setyp
-            :yellow
-          else
-            if status.to_s.index ">"
-              :cyan
-            else
-              :cyan
-            end
-          end
-  Log.color(color, status.to_s)
-end
-
-def report_msg(status, name, path, info = nil)
-  parts = path.sub(/\{.*/,'').sub(/#{Regexp.quote(name)}$/,'').split "/"
-
-  task = Log.color(:yellow, parts.pop)
-  workflow = Log.color(:magenta, parts.pop)
-  if status.to_s == 'noinfo' and parts.last != 'jobs'
-    task, status, workflow = Log.color(:yellow, info[:task_name]), Log.color(:blue, "file"), Log.color(:magenta, "-")
-  end
-
-  path_mtime = begin
-                   Open.mtime(path)
-               rescue
-                   nil
-               end
-  str = if not Open.remote?(path) and (Open.exists?(path) and $main_mtime and path_mtime and ($main_mtime - path_mtime) < -2)
-          iii [path, path_mtime, $main_mtime, $main_mtime - path_mtime]
-    status_msg(status.to_s) << " " << [workflow, task, path].compact * " " << " (#{Log.color(:red, "Mtime out of sync") })"
-  else
-    status_msg(status.to_s) << " " << [workflow, task, path].compact * " " 
-  end
-
-  if $inputs and $inputs.any? 
-    job_inputs = Workflow.load_step(path).recursive_inputs.to_hash
-    IndiferentHash.setup(job_inputs)
-
-    $inputs.each do |input|
-      value = job_inputs[input]
-      next if  value.nil?
-      value_str = Misc.fingerprint(value)
-      str << "\t#{Log.color :magenta, input}=#{value_str}"
-    end
-  end
-
-  if $info_fields and $info_fields.any?
-    $info_fields.each do |field|
-      IndiferentHash.setup(info)
-      value = info[field]
-      next if value.nil?
-      value_str = Misc.fingerprint(value)
-      str << "\t#{Log.color :magenta, field}=#{value_str}"
-    end
-  end
-
-  str << "\n"
-end
-
 def touch(step)
   return unless File.exists?(step.path)
   step.dependencies.each do |dep|
@@ -120,29 +53,6 @@ def touch(step)
   end if step.dependencies
 end
 
-def report(step, offset = 0, task = nil)
-  info = step.info  || {}
-  info[:task_name] = task
-  path  = step.path
-  status = info[:status] || :missing
-  status = "remote" if Open.remote?(path)
-  name = info[:name] || File.basename(path)
-  status = :unsync if status == :done and not Open.exist? path
-  str = " " * offset
-  str << report_msg(status, name, path, info)
-  step.dependencies.each do |dep|
-    path = dep.path
-    new = ! $seen.include?(path)
-    if new
-      $seen << path
-      str << report(dep, offset + 1, task)
-    else
-      str << Log.color(:blue, Log.uncolor(report(dep, offset+1, task)))
-    end
-  end if step.dependencies
-  str
-end
-
 step = get_step file
 $main_mtime = Open.exist?(step.path) ? Open.mtime(step.path) : nil
 
@@ -220,6 +130,6 @@ if options[:plot]
   end
 
 else
-  puts report(step).strip
+  puts Step.prov_report(step).strip
 end
 

data/share/rbbt_commands/workflow/task

@@ -364,6 +364,12 @@ begin
     exit 0
   end
 
+  if options.delete(:provenance)
+    require 'rbbt/workflow/provenance'
+    puts Step.prov_report(job)
+    exit 0
+  end
+
   if do_fork
     ENV["RBBT_NO_PROGRESS"] = "true"
     if detach
@@ -380,12 +386,6 @@ begin
   end
 
 
-  if options.delete(:provenance)
-    job.join
-    pp job.provenance_paths
-    exit 0
-  end
-
   if options.delete(:printname)
     job.join if IO === job.result
     puts job.name

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.25.38
+  version: 5.25.39
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-03-28 00:00:00.000000000 Z
+date: 2019-04-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -323,6 +323,7 @@ files:
 - lib/rbbt/workflow/definition.rb
 - lib/rbbt/workflow/doc.rb
 - lib/rbbt/workflow/examples.rb
+- lib/rbbt/workflow/provenance.rb
 - lib/rbbt/workflow/soap.rb
 - lib/rbbt/workflow/step.rb
 - lib/rbbt/workflow/step/dependencies.rb