rbbt-util 5.13.5 → 5.13.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 419096bee8e448dd3c606c3c9770d76f347ede9b
-  data.tar.gz: 85c4317ea460338198ee814424988f0a12026dd3
+  metadata.gz: 4cb8cbb3f115088d3b7a4fbe8fb9aa75cf4f4805
+  data.tar.gz: f61c90e38a18f4227f434260c83b6cf2ea48542a
 SHA512:
-  metadata.gz: 3b9cb44704148225709dc3ba2e8aaac7b586829849700347d7152e78ecc3bc2d55f73eb44429eb24038495109ea49a65154050a33f427245352de6feee9513c6
-  data.tar.gz: 9baaffb5b6eacc7781b503969098ff53bbfa4052deb846f2fbc4f592fc69e3932cc6290f39f754fd15a95f083883aaf80c5f8280c70cdcbd172bc4abbadb9101
+  metadata.gz: 25ac9c560885c7fb66acee8f693a6276f85f74fcaef12bfcc57c3ec59c3562bd3a1a8e25706dce34374c84f7186acaf9c2e901471e0c6002352182a5ecd7534b
+  data.tar.gz: b83ebb5dcdd346679f8a3299fa72ef9dd414590bf5775bcf574639b30be77844dea1bda0a7d6e4b74865992567fae452ed6943fdaaa8be76b503d96d80b37ef9

data/lib/rbbt/persist.rb

@@ -396,7 +396,7 @@ module Persist
 
     return (persist_options[:repo] || Persist::MEMORY)[persist_options[:file]] ||= yield if type ==:memory and persist_options[:file] and persist_options[:persist] and persist_options[:persist] != :update
 
-    if FalseClass == persist_options[:persist]
+    if FalseClass === persist_options[:persist]
       yield
     else
       other_options = Misc.process_options persist_options, :other

data/lib/rbbt/util/log/progress.rb

@@ -1,243 +1,49 @@
 require 'rbbt/util/log'
+require 'rbbt/util/log/progress/util'
+require 'rbbt/util/log/progress/report'
 module Log
   class ProgressBar
 
-    attr_accessor :depth, :num_reports, :desc, :io, :severity, :history, :max
-
-    # Creates a new instance. Max is the total number of iterations of the
-    # loop. The depth represents how many other loops are above this one,
-    # this information is used to find the place to print the progress
-    # report.
-    def initialize(max, options = {})
+    attr_accessor :max, :ticks, :frequency, :depth, :desc
+    def initialize(max = nil, options = {})
       options = Misc.add_defaults options, :depth => 0, :num_reports => 100, :desc => "Progress", :io => STDERR, :severity => Log.severity
       depth, num_reports, desc, io, severity = Misc.process_options options, :depth, :num_reports, :desc, :io, :severity
 
       @max = max
-      @max = 1 if @max and @max < 1
-      @current = 0
-      @time = Time.now
-      @last_report = -1
-      @num_reports = num_reports
-      @severity = severity
+      @ticks = 0
+      @frequency = 2
+      @last_time = nil
+      @last_count = nil
+      @last_percent = nil
       @depth = depth
       @desc = desc
     end
 
-    # Used to register a new completed loop iteration.
-    def tick(step = nil)
-      return if ENV["RBBT_NO_PROGRESS"] == 'true'
-
-      if step.nil?
-        @current += 1
-      else
-        @current = step
-      end
-
-      if @max
-        if percent - @last_report > 1.to_f/@num_reports.to_f 
-          report
-          @last_report=percent
-        end
-      else
-        if @last_report == -1 or Time.now - @last_report >= 1.0 
-          throughput
-        end
-      end
-
-      nil
-    end
-
-    def progress
-      @current.to_f/ @max
-    end
-
     def percent
-      (self.progress * 100).to_i
-    end
-
-    def eta
-      (Time.now - @time)/progress * (1-progress)
-    end
-
-    def used
-      (Time.now - @time).to_i
-    end
-
-
-    def up_lines(depth)
-      "\033[#{depth + 1}F\033[2K"
-    end
-
-    def down_lines(depth)
-      "\n\033[#{depth + 2}E"
-    end
-
-    def report_msg
-      progress = self.progress
-      percent = self.percent
-
-      indicator = Log.color(:magenta, @desc) << " "
-      10.times{|i|
-        if i < progress * 10 then
-          indicator << Log.color(:yellow, ".")
-        else
-          indicator << " "
-        end
-      }
-      done = progress == 1
-
-      used = self.used
-      used = [used/3600, used/60 % 60, used % 60].map{|t|  "%02i" % t }.join(':')
-
-      if progress == 1
-        indicator << Log.color(:green, " done")
-        indicator << Log.color(:blue, " #{used}")
-      else
-        indicator << " #{Log.color(:blue, percent.to_s << "%")}"
-
-        eta = self.eta
-        eta = [eta/3600, eta/60 % 60, eta % 60].map{|t| "%02i" % t }.join(':')
-
-        indicator << " #{Log.color :yellow, used} used #{Log.color :yellow, eta} left"
-      end
-
+      (@ticks * 100) / @max
     end
 
-    def thr
-      if @last_report == -1
-        @last_report = Time.now 
-        return thr
-      end
-      time = Time.now - @last_report
-      time = 0.000001 if time == 0
-      thr = (@current / time).to_i
+    def tick
+      @ticks += 1
 
-      if @history.nil?
-        @history ||= []
-      else
-        @history << thr
-        @history.shift if @history.length > 10
+      time = Time.now
+      if @last_time.nil?
+        @last_time = time
+        @last_count = @ticks
+        @start = time
+        return
       end
 
-      thr
-    end
+      diff = time - @last_time
+      report and return if diff > @frequency
+      return unless max
 
-    def mean
-      @mean_max ||= 0
-      if @history.length > 3
-        mean = Misc.mean(@history)
-        @mean_max = mean if mean > @mean_max
-      end
-      mean
-    end
-
-    def throughput_msg
-      thr = self.thr
-
-      mean = self.mean
-
-      indicator = Log.color(:magenta, @desc) 
-      indicator << " #{ Log.color :blue, thr } per second"
-
-      indicator << " #{ Log.color :yellow, mean.to_i } avg. #{ Log.color :yellow, @mean_max.to_i} max." if mean
-
-      indicator
-    end
-
-    def msg
-      @max ? report_msg : throughput_msg
-    end
-
-    def print(io, str)
-      LOG_MUTEX.synchronize do
-        STDERR.print str
-        Log.logfile.puts str unless Log.logfile.nil?
-        Log::LAST.replace "progress"
-      end
-    end
-    # Prints de progress report. It backs up as many lines as the meters
-    # depth. Prints the progress as a line of dots, a percentage, time
-    # spent, and time left. And then goes moves the cursor back to its
-    # original line. Everything is printed to stderr.
-    def report(io = STDERR)
-      if Log::LAST != "progress"
-        Log::LAST.replace "progress"
-        length = Log::ProgressBar.cleanup_bars
-        length.times{print(io, "\n")}
-      else
-        print(io, "\n") if @last_report == -1
-      end
-      print(io, up_lines(@depth) << report_msg << down_lines(@depth)) if severity >= Log.severity
-      @last_report = Time.now if @last_report == -1
-    end
-
-    def throughput(io = STDERR)
-      if Log::LAST != "progress"
-        Log::LAST.replace "progress"
-        length = Log::ProgressBar.cleanup_bars 
-        length.times{print(io, "\n")}
-      else
-        print(io, "\n") if @last_report == -1
-      end
-      print(io, up_lines(@depth) << throughput_msg << down_lines(@depth)) if severity >= Log.severity
-      @last_report = Time.now
-      @current = 0
-    end
-
-    def done
-      if Log::LAST != "progress"
-        Log::LAST.replace "progress"
-        length = Log::ProgressBar.cleanup_bars 
-        length.times{print(io, "\n")}
-      end
-      print(io, up_lines(@depth) << Log.color(:magenta, @desc) << Log.color(:green, " DONE") << down_lines(@depth)) if severity >= Log.severity
-    end
-
-    BAR_MUTEX = Mutex.new
-    BARS = []
-    REMOVE = []
-    def self.new_bar(max, options = {})
-      cleanup_bars
-      BAR_MUTEX.synchronize do
-        options = Misc.add_defaults options, :depth => BARS.length
-        BARS << (bar = ProgressBar.new(max, options))
-        bar
-      end
-    end
-
-    def self.cleanup_bars
-      BAR_MUTEX.synchronize do
-        REMOVE.each do |bar|
-          index = BARS.index bar
-          if index
-            BARS.delete_at index
-            BARS.each_with_index do |bar,i|
-              bar.depth = i
-            end
-          end
-        end
-        REMOVE.clear
-        BARS.length
-      end
-    end
-
-    def self.remove_bar(bar)
-      bar.done unless bar.max or ENV["RBBT_NO_PROGRESS"] == 'true'
-      BAR_MUTEX.synchronize do
-        REMOVE << bar
-      end
-    end
-
-    def self.with_bar(max, options = {})
-      bar = new_bar(max, options)
-      begin
-        yield bar
-        keep = false
-      rescue KeepBar
-        keep = true
-      ensure
-        remove_bar(bar) if bar
+      percent = self.percent
+      if @last_percent.nil?
+        @last_percent = percent
+        return
       end
+      report and return if percent > @last_percent and diff > 0.3
     end
   end
 end

data/lib/rbbt/util/log/progress/report.rb

@@ -0,0 +1,108 @@
+module Log
+  class ProgressBar
+    def up_lines(depth)
+      "\033[#{depth + 1}F\033[2K"
+    end
+
+    def down_lines(depth)
+      "\n\033[#{depth + 2}E"
+    end
+
+    def print(io, str)
+      LOG_MUTEX.synchronize do
+        STDERR.print str
+        Log.logfile.puts str unless Log.logfile.nil?
+        Log::LAST.replace "progress"
+      end
+    end
+
+    attr_accessor :history, :mean_max
+    def thr
+      count = @ticks - @last_count
+      seconds = Time.now - @last_time
+      thr = count / seconds
+    end
+
+    def thr_msg
+      thr = self.thr
+      if @history.nil?
+        @history ||= []
+      else
+        @history << thr
+        @history.shift if @history.length > 10
+      end
+
+      @mean_max ||= 0
+      if @history.length > 3
+        mean = Misc.mean(@history)
+        @mean_max = mean if mean > @mean_max
+      end
+
+      str = "#{ Log.color :blue, thr.to_i.to_s } per sec."
+      str << " #{ Log.color :yellow, mean.to_i.to_s } avg. #{Log.color :yellow, @mean_max.to_i.to_s} max." if @mean_max > 0
+      str
+    end
+
+    def eta_msg
+      percent = self.percent
+      time = Time.now
+
+      indicator = ""
+      10.times{|i|
+        if i < percent / 10 then
+          indicator << Log.color(:yellow, ".")
+        else
+          indicator << " "
+        end
+      }
+
+      indicator << " #{Log.color(:blue, percent.to_s << "%")}"
+
+      used = time - @start
+      if @mean_max and @mean_max > 0
+        eta =  (@max - @ticks) / @mean_max
+      else
+        eta =  (@max - @ticks) / (@ticks/used)
+      end
+
+      used = [used/3600, used/60 % 60, used % 60].map{|t|  "%02i" % t }.join(':')
+      eta = [eta/3600, eta/60 % 60, eta % 60].map{|t| "%02i" % t }.join(':')
+
+      indicator << " #{Log.color :yellow, used} used #{Log.color :yellow, eta} left"
+
+      indicator
+    end
+
+    def report_msg
+      str = Log.color :magenta, desc
+      return str << " " << Log.color(:yellow, "waiting") if @ticks == 0
+      str << " " << thr_msg
+      str << Log.color(:blue, " -- ") << eta_msg  if max
+      str
+    end
+
+    def report(io = STDERR)
+      if Log::LAST != "progress"
+        length = Log::ProgressBar.cleanup_bars
+        print(io, Log.color(:yellow, "--Progress\n"))
+        bars = BARS
+        bars.sort_by{|b| b.depth }.reverse.each do |bar|
+          print(io, Log.color(:yellow ,bar.report_msg) << "\n")
+        end
+      end
+      print(io, up_lines(@depth) << report_msg << down_lines(@depth)) 
+      @last_time = Time.now
+      @last_count = ticks
+      @last_percent = percent if max
+    end
+
+    def done(io = STDERR)
+      done_msg = Log.color(:magenta, desc) << " " << Log.color(:green, "done")
+      done_msg << " " << Log.color(:yellow, (@ticks).to_s) << " in " << Log.color(:blue, (Time.now - @start).to_i.to_s) << " sec." 
+      @last_count = 0
+      @last_time = @start
+      done_msg << " (" << thr_msg << ")"
+      print(io, up_lines(@depth) << done_msg << down_lines(@depth)) 
+    end
+  end
+end

data/lib/rbbt/util/log/progress/util.rb

@@ -0,0 +1,53 @@
+module Log
+  class ProgressBar
+    BAR_MUTEX = Mutex.new
+    BARS = []
+    REMOVE = []
+
+    def self.new_bar(max, options = {})
+      cleanup_bars
+      BAR_MUTEX.synchronize do
+        Log::LAST.replace "new_bar" if Log::LAST == "progress"
+        options = Misc.add_defaults options, :depth => BARS.length
+        BARS << (bar = ProgressBar.new(max, options))
+        bar
+      end
+    end
+
+    def self.cleanup_bars
+      BAR_MUTEX.synchronize do
+        REMOVE.each do |bar|
+          index = BARS.index bar
+          if index
+            BARS.delete_at index
+            BARS.each_with_index do |bar,i|
+              bar.depth = i
+            end
+          end
+        end
+        REMOVE.clear
+        BARS.length
+      end
+    end
+
+    def self.remove_bar(bar)
+      bar.done 
+      BAR_MUTEX.synchronize do
+        REMOVE << bar
+      end
+    end
+
+    def self.with_bar(max, options = {})
+      bar = new_bar(max, options)
+      begin
+        yield bar
+        keep = false
+      rescue KeepBar
+        keep = true
+      ensure
+        remove_bar(bar) if bar
+      end
+    end
+  end
+end
+

data/lib/rbbt/workflow/accessor.rb

@@ -85,6 +85,20 @@ class Step
     end
   end
 
+  def merge_info(hash)
+    return nil if @exec or info_file.nil?
+    value = Annotated.purge value if defined? Annotated
+    lock_filename = Persist.persistence_path(info_file, {:dir => Step.lock_dir})
+    Open.lock(info_file, :refresh => false) do
+      i = info
+      i.merge! hash
+      @info_cache = i
+      Open.write(info_file, INFO_SERIALIAZER.dump(i))
+      @info_cache_time = Time.now
+      value
+    end
+  end
+
   def status
     info[:status]
   end

data/lib/rbbt/workflow/step/run.rb

@@ -185,11 +185,14 @@ class Step
 
           Open.rm info_file if Open.exists? info_file
 
-          set_info :pid, Process.pid
-          set_info :issued, Time.now
+          log :setup, "#{Log.color :green, "Task"} #{Log.color :yellow, task.name.to_s || ""}"
 
-          log(:preparing, "Preparing job: #{Misc.fingerprint dependencies}")
-          set_info :dependencies, dependencies.collect{|dep| [dep.task_name, dep.name]}
+          merge_info({
+            :pid => Process.pid,
+            :issued => Time.now,
+            :name => name,
+            :dependencies => dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]},
+          })
 
           dup_inputs
           begin
@@ -391,13 +394,12 @@ class Step
   def abort
     return if @aborted
     @aborted = true
-    return if error?
     return if done?
     Log.medium{"#{Log.color :red, "Aborting"} #{Log.color :blue, path}"}
     begin
+      stop_dependencies
       abort_stream
       abort_pid
-      stop_dependencies
     rescue Aborted
       Log.medium{"#{Log.color :red, "Aborting ABORTED RETRY"} #{Log.color :blue, path}"}
       retry
@@ -427,11 +429,9 @@ class Step
     if stream
       begin
         Misc.consume_stream stream
-        stream.join if stream.respond_to? :join 
       rescue Exception
         self.abort
         raise $!
-        #stream.abort if stream.respond_to? :abort 
       end
     end
   end

data/share/rbbt_commands/alias

@@ -8,8 +8,12 @@ cmd, *rest = ARGV
 aliases ||= Rbbt.etc.cmd_alias.exists? ? Rbbt.etc.cmd_alias.yaml : {}
 
 if cmd.nil?
-  require 'pp'
-  pp aliases
+  aliases.each do |name, parts|
+    parts = parts.collect{|p|
+      p =~ /^[\w:_\/-]*$/ ? p : "'" << p << "'"
+    }
+    puts [Log.color(:magenta, name), parts * " "] * ": "
+  end
   exit 0
 end
 

data/share/rbbt_commands/workflow/prov

@@ -0,0 +1,65 @@
+#!/usr/bin/env ruby
+
+require 'rbbt/workflow'
+
+require 'rbbt-util'
+require 'fileutils'
+require 'rbbt/util/simpleopt'
+require 'rbbt/workflow/step'
+require 'rbbt/util/misc'
+
+file = ARGV.shift
+
+$seen = []
+def get_step(file)
+  file = file.sub(/\.(info|files)/,'')
+  $seen << file
+  Step.new file
+end
+
+def status_msg(status)
+  color = case status
+          when :error, :aborted, :missing
+            :red
+          when :streaming
+            :yellow
+          when :done
+            :green
+          else
+            nil
+          end
+  Log.color(color, status)
+end
+
+def report_msg(status, name, path)
+
+  parts = path.sub(/#{Regexp.quote(name)}$/,'').split "/"
+
+  task = Log.color(:yellow, parts.pop)
+  workflow = Log.color(:magenta, parts.pop)
+
+  status_msg(status) << " " << [workflow, task, path] * " " << "\n"
+end
+
+def report(step, offset = 0)
+  info = step.info || {}
+  path  = step.path
+  status = info[:status] || :missing
+  name = info[:name] || File.basename(path)
+  str = " " * offset
+  str << report_msg(status, name, path)
+  info[:dependencies].each do |task,name,path|
+    new = ! $seen.include?(path)
+    dep = get_step path
+    if new
+      str << report(dep, offset + 1)
+    else
+      str << Log.color(:blue, Log.uncolor(report(dep, offset+1)))
+    end
+  end if info[:dependencies]
+  str
+end
+
+step = get_step file
+
+puts report(step).strip

data/share/rbbt_commands/workflow/provenance

@@ -56,21 +56,31 @@ def print(file, offset = 1)
   dependencies = info[:dependencies] || []
   color = case
           when (not info)
-            Log::SEVERITY_COLOR[3]
+            nil
           when info[:status] == :error 
-            Log::SEVERITY_COLOR[3]
+            :red
           when info[:status] == :aborted 
-            Log::SEVERITY_COLOR[2]
+            :red
           when (info[:status] != :done and info[:pid] and not running? info)
-            Log::SEVERITY_COLOR[2]
+            :green
           end
 
-  puts (" " * offset * 2) << job_str(step.path, step.info, color)
+  puts (" " * offset * 2) << Log.color(color, step.path)
   $done << file
   root = File.dirname(File.dirname(file))
+  main_parts = step.path.split "/"
   dependencies.each do |task,new_file|
     new_file = File.join(root, task.to_s, new_file)
-    print(new_file, offset+1)
+    parts = new_file.split "/"
+    common ||= begin
+                 i = 0
+                 parts.each do |p|
+                   break unless main_parts[i] == p
+                   i += 1
+                 end
+                 i
+               end
+    print(new_file[common-1..-1], offset+1)
   end
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.13.5
+  version: 5.13.6
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-06 00:00:00.000000000 Z
+date: 2014-05-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -175,6 +175,8 @@ files:
 - lib/rbbt/util/filecache.rb
 - lib/rbbt/util/log.rb
 - lib/rbbt/util/log/progress.rb
+- lib/rbbt/util/log/progress/report.rb
+- lib/rbbt/util/log/progress/util.rb
 - lib/rbbt/util/misc.rb
 - lib/rbbt/util/misc/concurrent_stream.rb
 - lib/rbbt/util/misc/development.rb
@@ -257,6 +259,7 @@ files:
 - share/rbbt_commands/workflow/jobs
 - share/rbbt_commands/workflow/list
 - share/rbbt_commands/workflow/monitor
+- share/rbbt_commands/workflow/prov
 - share/rbbt_commands/workflow/provenance
 - share/rbbt_commands/workflow/remote/add
 - share/rbbt_commands/workflow/remote/list