rbbt-util 5.21.126 → 5.21.127

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b25c57ba759c64c90f84606b1903060f3551cf20
-  data.tar.gz: d11206948af6273c56a6ac051a1ca3e064214a72
+  metadata.gz: 2b39b4950994d49ab0541d19a8e7d3ac45245c1e
+  data.tar.gz: 28ac4746e480b19a5c9005671f15a29a97586f46
 SHA512:
-  metadata.gz: a79b14310fb16641876d4b84629f3af35a783a42c7785019ff7bdd9c44492930f80c3293e36fb966b66e4ff86c668a2823c48465da7ad011e0bf7bb6958e625b
-  data.tar.gz: 78ef5d0e917fe5c5be16b453cb94a757397c46c3d20d38fe2a67b0c2ea26800cdb3f33e0e223444b0b5c2371e173c0d6a654f5005d9f0e6b21997178bd56fc49
+  metadata.gz: 1722d689b494dedf1095a53f288b063dd12c2aa1ce01a05c3e92b17172266facd7445879d10e75d7aac65ebe22c502d37398e9345e19ffaf7297d680400b84df
+  data.tar.gz: 57e25447236ba7c6e5f28be95c687513c1259f742ddbdbe0d7957b695b9c17177e3c0105c821638871afec07e2249a3a9b2ff6c5015c9288ff0c018664fc4765

data/lib/rbbt/monitor.rb

@@ -146,7 +146,7 @@ module Rbbt
           task = File.basename(taskdir)
           next if tasks and not tasks.include? task
 
-          cmd = "find -L '#{ taskdir }/' -not -type d -not -path '*/*.files/*' -not -path '*/*.pid' 2>/dev/null"
+          cmd = "find -L '#{ taskdir }/' -not \\( -path \"#{taskdir}/*.files\" -prune \\) -not -path '*/*.pid' -not -type d 2>/dev/null"
           files = CMD.cmd(cmd, :pipe => true)
           TSV.traverse files, :type => :array, :into => jobs, :_bar => "Finding jobs in #{ taskdir }" do |file|
             _files << file

data/lib/rbbt/workflow.rb

@@ -4,6 +4,7 @@ require 'rbbt/workflow/step'
 require 'rbbt/workflow/accessor'
 require 'rbbt/workflow/doc'
 require 'rbbt/workflow/examples'
+require 'rbbt/workflow/archive'
 
 module Workflow
 

data/lib/rbbt/workflow/accessor.rb

@@ -813,8 +813,10 @@ module Workflow
                        next if d.nil?
                        if Hash === d
                          d[:workflow] ||= wf 
-                         d[:task] = task_name
-                         inputs = assign_dep_inputs({}, options.merge(d[:inputs] || {}), real_dependencies, d[:workflow].task_info(d[:task])) 
+                         d[:task] ||= task_name
+                         task_info = d[:workflow].task_info(d[:task])
+
+                         inputs = assign_dep_inputs({}, options.merge(d[:inputs] || {}), real_dependencies, task_info) 
                          d = d[:workflow].job(d[:task], d[:jobname], inputs) 
                        end
                        ComputeDependency.setup(d, compute) if compute

data/lib/rbbt/workflow/archive.rb

@@ -0,0 +1,49 @@
+class Step
+  def self.link_job(path, target_dir, task = nil, workflow = nil)
+    Path.setup(target_dir)
+
+    name = File.basename(path)
+    task = File.basename(File.dirname(path)) if task.nil?
+    workflow = File.basename(File.dirname(File.dirname(path))) if workflow.nil?
+
+    FileUtils.mkdir_p target_dir[workflow][task]
+    FileUtils.ln_s path, target_dir[workflow][task][name].find
+    FileUtils.ln_s path + '.files', target_dir[workflow][task][name].find + '.files' if File.exists?(path + '.files')
+    FileUtils.ln_s path + '.info', target_dir[workflow][task][name].find + '.info' if File.exists?(path + '.info')
+  end
+
+  def archive(target = nil)
+    target = self.path + '.tar.gz' if target.nil?
+    target = File.expand_path(target)
+    TmpFile.with_file do |tmpdir|
+      Step.link_job self.path, tmpdir
+      rec_dependencies = Set.new
+      deps = [self.path]
+      seen = Set.new
+      while deps.any?
+        path = deps.shift
+        dep = Step.new path
+        seen << dep.path
+        dep.info[:dependencies].each do |task, name, path|
+          dep = Step.new path
+          next if seen.include? dep.path
+          deps << dep.path
+          rec_dependencies << dep.path
+        end if dep.info[:dependencies]
+      end
+
+      rec_dependencies.each do |path|
+        Step.link_job path, tmpdir
+      end
+
+      Misc.in_dir(tmpdir) do
+        io = CMD.cmd("tar cvhzf '#{target}'  ./*", :pipe => true)
+        while line = io.gets
+          Log.debug line
+        end
+        io.join if io.respond_to? :join
+      end
+      Log.debug "Archive finished at: #{target}"
+    end
+  end
+end

data/lib/rbbt/workflow/examples.rb

@@ -51,9 +51,8 @@ module Workflow
 
   def example_inputs(task_name, example)
     inputs = {}
-    IndiferentHash.setup(input)
+    IndiferentHash.setup(inputs)
     example(task_name, example).each do |input,type,file|
-      next if new_inputs.include? input
 
       case type
       when :tsv, :array, :text

data/share/rbbt_commands/system/status

@@ -50,14 +50,16 @@ def status_msg(status)
           when :error, :aborted, :missing, :dead
             :red
           when :streaming, :started
-            :yellow
+            :cyan
           when :done
             :green
+          when :dependencies, :waiting, :setyp
+            :yellow
           else
             if status.to_s.index ">"
-              :yellow
+              :cyan
             else
-              nil
+              :cyan
             end
           end
   Log.color(color, status.to_s)

data/share/rbbt_commands/workflow/archive

@@ -0,0 +1,31 @@
+#!/usr/bin/env ruby
+
+require 'rbbt/workflow'
+
+require 'rbbt-util'
+require 'fileutils'
+require 'rbbt/util/simpleopt'
+require 'rbbt/workflow/step'
+require 'rbbt/util/misc'
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+
+$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+Examine the provenance of a job result
+
+$ rbbt workflow archive <job> [target]
+
+Archive job and all dependencies into the target tar gz file. If no target is specified 
+it defaults to a <job>.tar.gz
+
+-h--help Help
+EOF
+
+SOPT.usage if options[:help]
+
+file, target = ARGV
+
+Step.new(File.expand_path(file)).archive(target)

data/share/rbbt_commands/workflow/info

@@ -31,17 +31,23 @@ def get_step(file)
 end
 
 def status_msg(status)
-  color = case status
-          when :error, :aborted, :missing
+  color = case status.to_sym
+          when :error, :aborted, :missing, :dead
             :red
-          when :streaming
-            :yellow
+          when :streaming, :started
+            :cyan
           when :done
             :green
+          when :dependencies, :waiting, :setyp
+            :yellow
           else
-            nil
+            if status.to_s.index ">"
+              :cyan
+            else
+              :cyan
+            end
           end
-  Log.color(color, status)
+  Log.color(color, status.to_s)
 end
 
 def pid_msg(pid, done = false)

data/share/rbbt_commands/workflow/prov

@@ -34,15 +34,23 @@ def get_step(file)
 end
 
 def status_msg(status)
-  color = case status
-          when :error, :aborted, :missing
+  color = case status.to_sym
+          when :error, :aborted, :missing, :dead
             :red
-          when :streaming
-            :yellow
+          when :streaming, :started
+            :cyan
           when :done
             :green
+          when :dependencies, :waiting, :setyp
+            :yellow
+          else
+            if status.to_s.index ">"
+              :cyan
+            else
+              :cyan
+            end
           end
-  Log.color(color, status)
+  Log.color(color, status.to_s)
 end
 
 def report_msg(status, name, path)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.21.126
+  version: 5.21.127
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-03-01 00:00:00.000000000 Z
+date: 2018-03-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -316,6 +316,7 @@ files:
 - lib/rbbt/workflow.rb
 - lib/rbbt/workflow/accessor.rb
 - lib/rbbt/workflow/annotate.rb
+- lib/rbbt/workflow/archive.rb
 - lib/rbbt/workflow/definition.rb
 - lib/rbbt/workflow/doc.rb
 - lib/rbbt/workflow/examples.rb
@@ -387,6 +388,7 @@ files:
 - share/rbbt_commands/tsv/write_excel
 - share/rbbt_commands/tsv/zip
 - share/rbbt_commands/watch
+- share/rbbt_commands/workflow/archive
 - share/rbbt_commands/workflow/cmd
 - share/rbbt_commands/workflow/example
 - share/rbbt_commands/workflow/info