rbbt-util 5.28.14 → 5.30.0

data/lib/rbbt/persist.rb

@@ -110,6 +110,8 @@ module Persist
   def self.load_file(path, type)
     begin
       case (type || :marshal).to_sym
+      when :path
+        path
       when :nil
         nil
       when :boolean
@@ -167,6 +169,8 @@ module Persist
     end
 
     case (type || :marshal).to_sym
+    when :path
+      nil
     when :nil
       nil
     when :boolean

data/lib/rbbt/persist/tsv/adapter.rb

@@ -104,9 +104,6 @@ module Persist
         write(true) if closed? || ! write?
         res = begin
                 yield
-              rescue Exception
-                Log.exception $!
-                raise $!
               ensure
                 close
               end
@@ -115,7 +112,6 @@ module Persist
     end
 
     def read_and_close
-      #return yield if @locked
       if read? || write?
         begin
           return yield
@@ -134,6 +130,45 @@ module Persist
       end
     end
 
+    def read_lock
+      read if closed?
+      if read?
+        begin
+          return yield
+        ensure
+          close
+        end
+      end
+
+      lock do
+        close
+        read true
+        begin
+          yield
+        end
+      end
+    end
+
+    def write_lock
+      write if closed?
+      if write?
+        begin
+          return yield
+        ensure
+          close
+        end
+      end
+
+      lock do
+        close
+        write true
+        begin
+          yield
+        end
+      end
+    end
+
+
     def merge!(hash)
       hash.each do |key,values|
         self[key] = values
@@ -141,38 +176,38 @@ module Persist
     end
 
     def range(*args)
-      self.read_and_close do
+      self.read_lock do
         super(*args)
       end
     end
 
     def include?(*args)
-      self.read_and_close do
+      self.read_lock do
         super(*args) #- TSV::ENTRY_KEYS.to_a
       end
     end
 
     def [](*args)
-      self.read_and_close do
+      self.read_lock do
         super(*args) #- TSV::ENTRY_KEYS.to_a
       end
     end
 
     def []=(*args)
-      self.write_and_close do
+      self.write_lock do
         super(*args) #- TSV::ENTRY_KEYS.to_a
       end
     end
 
     def keys(*args)
-      self.read_and_close do
+      self.read_lock do
         super(*args)
       end
     end
 
 
     def prefix(key)
-      self.read_and_close do
+      self.read_lock do
         range(key, 1, key + MAX_CHAR, 1)
       end
     end
@@ -184,13 +219,13 @@ module Persist
 
 
     def size(*args)
-      self.read_and_close do
+      self.read_lock do
         super(*args)
       end
     end
 
     def each(*args, &block)
-      self.read_and_close do
+      self.read_lock do
         super(*args, &block)
       end
     end
@@ -208,7 +243,7 @@ module Persist
     end
 
     def values_at(*keys)
-      self.read_and_close do
+      self.read_lock do
         keys.collect do |k|
           self[k]
         end

data/lib/rbbt/util/cmd.rb

@@ -217,7 +217,7 @@ module CMD
     end
   end
 
-  def self.cmd_log(*args)
+  def self.cmd_pid(*args)
     all_args = *args
 
     all_args << {} unless Hash === all_args.last
@@ -248,4 +248,9 @@ module CMD
     nil
   end
 
+  def self.cmd_log(*args)
+    cmd_pid(*args)
+    nil
+  end
+
 end

data/lib/rbbt/util/misc/inspect.rb

@@ -287,7 +287,9 @@ module Misc
           when Symbol 
             obj.to_s
           when (defined?(Path) and Path)
-            if Step === obj.resource
+            if defined?(Step) && Open.exists?(Step.info_file(obj))
+              obj2str(Workflow.load_step(obj))
+            elsif defined?(Step) && Step === obj.resource
               "Step file: " + obj
             else
               if obj.exists?
@@ -322,7 +324,11 @@ module Misc
             remove_long_items(obj)
           when File 
             if obj.respond_to? :filename and obj.filename
-              "<IO:" << obj.filename << "--" << mtime_str(obj.filename) << ">"
+              if defined?(Step) && Open.exists?(Step.info_file(obj.filename))
+                obj2str(Workflow.load_step(obj.filename))
+              else
+                "<IO:" << obj.filename << "--" << mtime_str(obj.filename) << ">"
+              end
             else
               "<IO:" << obj.path << "--" << mtime_str(obj.path) << ">"
             end
@@ -330,7 +336,11 @@ module Misc
             "<IO:" << obj.short_path << ">"
           when IO
             if obj.respond_to? :filename and obj.filename
-              "<IO:" << obj.filename << "--" << mtime_str(obj.filename) << ">"
+              if defined?(Step) && Open.exists?(Step.info_file(obj.filename))
+                obj2str(Workflow.load_step(obj.filename))
+              else
+                "<IO:" << obj.filename << "--" << mtime_str(obj.filename) << ">"
+              end
             else
 
               if obj.respond_to? :obj2str

data/lib/rbbt/util/misc/options.rb

@@ -242,48 +242,6 @@ module Misc
 
     return options
 
-    options = {}
-    string.split(/#/).each do |str|
-      if str.match(/(.*)=(.*)/)
-        option, value = $1, $2
-      else
-        option, value = str, true
-      end
-
-      option = option.sub(":",'').to_sym if option.chars.first == ':'
-      value  = value.sub(":",'').to_sym if String === value and value.chars.first == ':'
-
-      if value == true
-        options[option] = option.to_s.chars.first != '!' 
-      else
-        options[option] = Thread.start do
-          $SAFE = 0;
-          case 
-          when value =~ /^(?:true|T)$/i
-            true
-          when value =~ /^(?:false|F)$/i
-            false
-          when Symbol === value
-            value
-          when (String === value and value =~ /^\/(.*)\/$/)
-            Regexp.new /#{$1}/
-          else
-            begin
-              Kernel.const_get value
-            rescue
-              begin  
-                raise if value =~ /[a-z]/ and defined? value
-                eval(value) 
-              rescue Exception
-                value 
-              end
-            end
-          end
-        end.value
-      end
-    end
-
-    options
   end
 
 end

data/lib/rbbt/util/procpath.rb

@@ -0,0 +1,49 @@
+require 'rbbt/util/cmd'
+module ProcPath
+  CMD.tool :procpath do
+    'pip install procpath'
+  end
+
+  def self.record(pid, path, options = {})
+    IndiferentHash.setup(options)
+    options = Misc.add_defaults options, "interval" => 30
+
+    cmd_options = %w(interval recnum reevalnum).inject({}){|acc,k| acc[k] = options[k]; acc}
+
+    Log.debug "ProcPath recording #{pid} in #{path} (#{Misc.fingerprint options})"
+    procpath_thread = Thread.new do 
+      begin
+        procpath_pid = CMD.cmd_pid(:procpath, "record --database-file '#{path}' '$..children[?(@.stat.pid == #{pid})]'", cmd_options.merge(:nofail => true, :add_option_dashes => true))
+      rescue Exception
+        Log.exceptions $!
+        Process.kill "INT", procpath_pid
+      end
+    end
+
+    procpath_thread.report_on_exception = false
+
+    Process.wait pid.to_i
+    procpath_thread.raise Interrupt
+  end
+
+  def self.plot(path, output, options = {})
+    IndiferentHash.setup(options)
+    options = Misc.add_defaults options, "query-name" => 'rss', 'epsilon' => 0.5, "moving-average-window" => 10
+
+    cmd_options = %w(query-name epsilon monitor-average-window title logarithmic after before custom-query-file custom-value-expr).inject({}){|acc,k| acc[k] = options[k]; acc}
+    CMD.cmd_log(:procpath, "plot --database-file '#{path}' --plot-file '#{output}' ", cmd_options.merge(:nofail => true, :add_option_dashes => true))
+  end
+
+  def self.monitor(pid, path)
+    database, options_str = path.split("#")
+    options = options_str.nil? ? {} : Misc.string2hash(options_str)
+
+    database = File.expand_path database
+    Log.low "ProcPath monitor #{pid} in #{database} (#{Misc.fingerprint options})"
+
+    ProcPath.record(pid, database + '.sqlite3', options)
+    ProcPath.plot(database + '.sqlite3', database + '.cpu.svg', options.merge("query-name" => 'cpu'))
+    ProcPath.plot(database + '.sqlite3', database + '.rss.svg', options.merge("query-name" => 'rss'))
+  end
+end
+

data/lib/rbbt/workflow.rb

@@ -385,7 +385,7 @@ module Workflow
       next if default == v 
       next if (String === default and Symbol === v and v.to_s == default)
       next if (Symbol === default and String === v and v == default.to_s)
-      real_inputs[k] = v 
+      real_inputs[k.to_sym] = v 
     end
 
     jobname_input_value = inputs[jobname_input] || all_defaults[jobname_input]
@@ -410,6 +410,7 @@ module Workflow
     job.workflow = self
     job.clean_name = jobname
     job.overriden = overriden
+    job.real_inputs = real_inputs.keys
     job
   end
 

data/lib/rbbt/workflow/accessor.rb

@@ -16,6 +16,10 @@ end
 
 module Workflow
 
+  def self.job_path?(path)
+    path.split("/")[-4] == "jobs"
+  end
+
   def log(status, message = nil, &block)
     Step.log(status, message, nil, &block)
   end
@@ -301,7 +305,9 @@ module Workflow
 
   def setup_override_dependency(dep, workflow, task_name)
     dep = Step === dep ? dep : Workflow.load_step(dep)
+    dep.workflow = workflow
     dep.info[:name] = dep.name
+    dep.original_task_name ||= dep.task_name if dep.workflow
     begin
       workflow = Kernel.const_get workflow if String === workflow
       dep.task = workflow.tasks[task_name] if dep.task.nil? && workflow.tasks.include?(task_name)
@@ -309,7 +315,7 @@ module Workflow
       Log.exception $!
     end
     dep.task_name = task_name
-    dep.overriden = true
+    dep.overriden = dep.original_task_name.to_sym
     dep
   end
 

data/lib/rbbt/workflow/examples.rb

@@ -50,8 +50,8 @@ module Workflow
         case input_types[input]
         when :file
           Log.debug "Pointing #{ input } to #{file}"
-          if file =~ /\.read$/
-            inputs[input.to_sym]  = Open.read(file)
+          if file =~ /\.yaml/
+            inputs[input.to_sym]  = YAML.load(Open.read(file))
           else
             inputs[input.to_sym]  = Open.realpath(file)
           end

data/lib/rbbt/workflow/step.rb

@@ -12,6 +12,7 @@ class Step
   attr_accessor :exec
   attr_accessor :relocated
   attr_accessor :result, :mutex, :seen
+  attr_accessor :real_inputs, :original_task_name
 
   RBBT_DEBUG_CLEAN = ENV["RBBT_DEBUG_CLEAN"] == 'true'
 
@@ -145,11 +146,13 @@ class Step
     seen = []
     while path = deps.pop
       dep_info = archived_info[path]
-      dep_info[:inputs].each do |k,v|
-        all_inputs[k] = v unless all_inputs.include?(k)
-      end if dep_info[:inputs]
-      deps.concat(dep_info[:dependencies].collect{|p| p.last } - seen) if dep_info[:dependencies]
-      deps.concat(dep_info[:archived_dependencies].collect{|p| p.last } - seen) if dep_info[:archived_dependencies]
+      if dep_info
+        dep_info[:inputs].each do |k,v|
+          all_inputs[k] = v unless all_inputs.include?(k)
+        end if dep_info[:inputs]
+        deps.concat(dep_info[:dependencies].collect{|p| p.last } - seen) if dep_info[:dependencies]
+        deps.concat(dep_info[:archived_dependencies].collect{|p| p.last } - seen) if dep_info[:archived_dependencies]
+      end
       seen << path
     end
 

data/lib/rbbt/workflow/step/accessor.rb

@@ -93,18 +93,22 @@ class Step
 
       Log.debug "Saving job input #{name} (#{type}) into #{path}"
       case
+      when Step === value
+        Open.ln_s(value.path, path)
+      when type.to_s == "file"
+        if String === value && File.exists?(value)
+          Open.ln_s(value, path)
+        else
+          Open.write(path + '.yaml', value.to_yaml)
+        end
       when Array === value
-        Open.write(path, value * "\n")
+        Open.write(path, value.collect{|v| Step === v ? v.path : v.to_s} * "\n")
       when IO === value
-        Open.write(path, value)
-      when type == "file"
-        if String === value && File.exists?(value)
-          Open.link(value, path)
+        if value.filename && String === value.filename && File.exists?(value.filename)
+          Open.ln_s(value.filename, path)
         else
-          Open.write(path + '.read', value.to_s)
+          Open.write(path, value)
         end
-      when Step === value
-        value = value.produce.load 
       else
         Open.write(path, value.to_s)
       end
@@ -114,18 +118,24 @@ class Step
   def self.save_job_inputs(job, dir, options = nil)
     options = IndiferentHash.setup options.dup if options
 
-    task_name = job.task_name
+    task_name = Symbol === job.overriden ? job.overriden : job.task_name
     workflow = job.workflow
     workflow = Kernel.const_get workflow if String === workflow
-    task_info = workflow.task_info(task_name)
-    input_types = task_info[:input_types]
-    task_inputs = task_info[:inputs]
-    input_defaults = task_info[:input_defaults]
+    if workflow
+      task_info = workflow.task_info(task_name)
+      input_types = task_info[:input_types]
+      task_inputs = task_info[:inputs]
+      input_defaults = task_info[:input_defaults]
+    else
+      task_info = input_types = task_inputs = input_defaults = {}
+    end
 
     inputs = {}
+    real_inputs = job.real_inputs || job.info[:real_inputs]
     job.recursive_inputs.zip(job.recursive_inputs.fields).each do |value,name|
       next unless task_inputs.include? name.to_sym
-      next if options and ! options.include?(name)
+      next unless real_inputs.include? name.to_sym
+      next if options && ! options.include?(name)
       next if value.nil?
       next if input_defaults[name] == value
       inputs[name] = value
@@ -222,7 +232,7 @@ class Step
   def init_info(force = false)
     return nil if @exec || info_file.nil? || (Open.exists?(info_file) && ! force)
     Open.lock(info_file, :lock => info_lock) do
-      i = {:status => :waiting, :pid => Process.pid, :path => path}
+      i = {:status => :waiting, :pid => Process.pid, :path => path, :real_inputs => real_inputs}
       i[:dependencies] = dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]} if dependencies
       Misc.sensiblewrite(info_file, Step.serialize_info(i), :force => true, :lock => false)
       @info_cache = IndiferentHash.setup(i)
@@ -495,8 +505,8 @@ class Step
 
   def running? 
     return false if ! (started? || status == :ending)
-    pid = info[:pid]
-    return nil if pid.nil?
+    return nil unless Open.exist?(self.pid_file)
+    pid = Open.read(self.pid_file).to_i
 
     return false if done? or error? or aborted? 
 
@@ -520,8 +530,7 @@ class Step
   end
 
   def nopid?
-    pid = info[:pid] || Open.exists?(pid_file)
-    ! pid && ! (status.nil? || status == :aborted || status == :done || status == :error || status == :cleaned)
+    ! Open.exists?(pid_file) && ! (status.nil? || status == :aborted || status == :done || status == :error || status == :cleaned)
   end
 
   def aborted?