rbbt-util 5.13.25 → 5.13.26

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5239dbb15f48c166b31834d3f927f4c46103dd2d
-  data.tar.gz: eef931195c348851b6aeb3de3ba35f4e043884f6
+  metadata.gz: 109a367647ca3e97bdd9cd9c7951f1dea64045d5
+  data.tar.gz: 0fc762047146427ce7dd94eddb4a8b7a05616cda
 SHA512:
-  metadata.gz: 60e3af52f609fce5ac4ef7c129bc31dff0bf06576bad4b10e2dabb9840388240bc4556f01eb045b24a0bd4206b8e7dde95566d3bb7fd01d88153e37f21f9e1ed
-  data.tar.gz: 50a0e08c13bb120b2b111de91f6446eb9d42cd0b6ca1897e8a2308e1f547cf2bb7d4d1198eeecf89ce2c2bf457456e3234403726cbdc314c2dcd8519187ee33f
+  metadata.gz: 8be51c3acb036f80bba3c2cb9082244eecb1dc06892da9322ff8b1d813f99b2233bb34176590ae671a3632d031cc2c6220568f19829c4c9701907c7ec5bf0d76
+  data.tar.gz: 55a3431a1e466e2b0aeb7d742592d9a24f9a69c1c4c15697c33b675dd4cd87c3f981627c476ef38adb85100a6b811d0329bd784a31c7faa96461cd0dfd541c57

data/bin/rbbt

@@ -19,8 +19,8 @@ if dev_dir.nil?
 end
 
 if dev_dir
-  Dir.glob(File.join(File.expand_path(dev_dir),'rbbt-*')).each do |f|
-    $LOAD_PATH << f
+  Dir.glob(File.join(File.expand_path(dev_dir),'rbbt-*/lib')).each do |f|
+    $LOAD_PATH.unshift f
   end
 end
 

data/lib/rbbt/monitor.rb

@@ -0,0 +1,83 @@
+require 'rbbt'
+
+module Rbbt
+
+  LOCK_DIRS    = Rbbt.share.find_all  + Rbbt.var.cache.persistence.find_all + Rbbt.tmp.tsv_open_locks.find_all + Rbbt.var.jobs.find_all 
+  def self.locks(dirs = LOCK_DIRS)
+    dirs.collect do |dir|
+      dir.glob("**/*.lock")
+    end.flatten
+  end
+
+  SENSIBLE_WRITE_DIRS = Misc.sensiblewrite_dir.find_all
+  def self.sensiblewrites(dirs = SENSIBLE_WRITE_DIRS)
+    dirs.collect do |dir|
+      dir.glob("**/*").reject{|f| f =~ /\.lock$/ }
+    end.flatten
+  end
+
+  PERSIST_DIRS    = Rbbt.share.find_all  + Rbbt.var.cache.persistence.find_all  
+  def self.persists(dirs = PERSIST_DIRS)
+    dirs.collect do |dir|
+      dir.glob("**/*.persist").reject{|f| f =~ /\.lock$/ }
+    end.flatten
+  end
+
+  JOB_DIRS = Rbbt.var.jobs.find_all
+  def self.jobs(dirs = JOB_DIRS)
+    job_files = {}
+    dirs.each do |dir|
+      workflow_dirs = dir.glob("*").each do |wdir|
+        workflow = File.basename(wdir)
+        job_files[workflow] = {}
+        task_dirs = wdir.glob('*')
+        task_dirs.each do |tdir|
+          task = File.basename(tdir)
+          job_files[workflow][task] = tdir.glob('*')
+        end
+      end
+    end
+    jobs = {}
+    job_files.each do |workflow,task_jobs|
+      jobs[workflow] ||= {}
+      task_jobs.each do |task, files|
+        jobs[workflow][task] ||= {}
+        files.each do |f|
+          next if f =~ /\.lock$/
+            job = f.sub(/\.(info|files)/,'')
+
+          jobs[workflow][task][job] ||= {}
+          if jobs[workflow][task][job][:status].nil? 
+            status = nil
+            status = :done if Open.exists? job
+            if status.nil? and f=~/\.info/
+              info = begin
+                       Step::INFO_SERIALIAZER.load(Open.read(f, :mode => 'rb'))
+                     rescue
+                       {}
+                     end 
+            status = info[:status]
+            pid = info[:pid]
+            end
+
+            jobs[workflow][task][job][:pid] = pid if pid
+            jobs[workflow][task][job][:status] = status if status
+          end
+        end
+      end
+    end
+    jobs
+  end
+
+  def self.load_lock(lock)
+    begin
+      info = Misc.insist 3 do 
+        YAML.load(Open.read(lock))
+      end
+      info.values_at "pid", "ppid", "time"
+    rescue Exception
+      [nil, nil, File.atime(lock)]
+    end
+  end
+
+end

data/lib/rbbt/util/log.rb

@@ -53,8 +53,12 @@ module Log
 
   self.ignore_stderr do
     require 'nokogiri'
-    require "highline/system_extensions.rb"
-    self.tty_size = HighLine::SystemExtensions.terminal_size.first 
+    self.tty_size = begin
+                      require "highline/system_extensions.rb"
+                      HighLine::SystemExtensions.terminal_size.first 
+                    rescue Exception
+                      nil
+                    end
   end
 
   def self.with_severity(level)

data/lib/rbbt/util/misc/concurrent_stream.rb

@@ -96,7 +96,7 @@ module ConcurrentStream
       @aborted = false if t == Thread.current
       next if t == Thread.current
       Log.medium "Aborting thread #{t.inspect}"
-      t.raise Aborted.new "Hey"
+      t.raise Aborted.new "From step: #{path}"
     end if @threads
 
     sleeped = false

data/share/rbbt_commands/system/clean

@@ -0,0 +1,96 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+require 'rbbt/workflow'
+require 'rbbt/monitor'
+
+$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Clean failed jobs, lock files, etc
+
+$ rbbt system clean 
+
+-h--help Print this help
+EOF
+rbbt_usage and exit 0 if options[:help]
+
+locks = Rbbt.locks
+sensiblewrites = Rbbt.sensiblewrites
+persists = Rbbt.persists
+
+puts Log.color(:magenta, "# System clean")
+puts
+if locks.any?
+  puts Log.color(:magenta, "Locks: #{locks.length}")
+  locks.each do |lock|
+    pid, ppid, time = Rbbt.load_lock(lock)
+    if not Misc.pid_exists? pid
+      puts "  Removing #{ lock }"
+      FileUtils.rm lock 
+    end
+  end
+  puts
+end
+
+if persists.any?
+  puts Log.color(:magenta, "Persist: #{persists.length}")
+  persists.each do |persist|
+    lf = persist + '.lock'
+    pid, ppid, time = Rbbt.load_lock(lf)
+    if not Misc.pid_exists? pid
+      puts "  Removing #{ persist }"
+      FileUtils.rm persists 
+      FileUtils.rm lf 
+    end
+  end
+  puts
+end
+
+if sensiblewrites.any?
+  puts Log.color(:magenta, "Writing: #{sensiblewrites.length}")
+  sensiblewrites.each do |sensiblewrite|
+    lf = sensiblewrite + '.lock'
+    pid, ppid, time = Rbbt.load_lock(lf)
+    if not Misc.pid_exists? pid
+      puts "  Removing #{ sensiblewrite }"
+      FileUtils.rm sensiblewrite 
+      FileUtils.rm lf 
+    end
+  end
+  puts
+end
+
+jobs = Rbbt.jobs
+
+puts Log.color(:magenta, "# Workflows:")
+puts
+jobs.each do |workflow, tasks|
+  tasks.each do |task, jobs|
+    puts "* " << Log.color(:magenta, workflow) << "#" << Log.color(:yellow, task) << ": #{jobs.length}"
+    done = []
+    other = {}
+    jobs.each do |file,h|
+      status = h[:status]
+      if h[:status] == :done
+        done << file
+      else
+        other[status||"missing"] ||= []
+        other[status||"missing"] << [file, h[:pid]]
+      end
+    end
+    other.each do |status, list|
+      list.each do |f,p|
+        if not Misc.pid_exists?(p) or status.to_s =~ /error|aborted|missing/
+          puts "  Removing #{ f }"
+          FileUtils.rm_rf f
+          FileUtils.rm_rf f + '.info'
+          FileUtils.rm_rf f + '.files'
+        end
+      end
+    end
+  end
+end
+

data/share/rbbt_commands/system/status

@@ -0,0 +1,105 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+require 'rbbt/workflow'
+require 'rbbt/monitor'
+
+$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Report the status of the system
+
+$ rbbt system status 
+
+-h--help Print this help
+EOF
+rbbt_usage and exit 0 if options[:help]
+
+
+def pid_msg(pid)
+  color = if Misc.pid_exists? pid
+            :green
+          else
+            :red
+          end
+  Log.color(color, pid)
+end
+
+def status_msg(status)
+  color = case status
+          when :error, :aborted, :missing
+            :red
+          when :streaming, :started
+            :yellow
+          when :done
+            :green
+          else
+            if status.to_s.index ">"
+              :yellow
+            else
+              nil
+            end
+          end
+  Log.color(color, status)
+end
+
+locks = Rbbt.locks
+sensiblewrites = Rbbt.sensiblewrites
+persists = Rbbt.persists
+
+puts Log.color(:magenta, "# System report")
+puts
+if locks.any?
+  puts Log.color(:magenta, "Locks:")
+  locks.each do |lock|
+    pid, ppid, time = Rbbt.load_lock(lock)
+    time ||= File.atime lock
+    puts "  " << lock + Log.color(:blue, " -- time: #{Time.now - time}; ppid: #{ppid}; pid: #{pid_msg pid}")
+  end
+  puts
+end
+
+if persists.any?
+  puts Log.color(:magenta, "Persist:")
+  persists.each do |persist|
+    puts "  " << persist
+  end
+  puts
+end
+
+if sensiblewrites.any?
+  puts Log.color(:magenta, "Writing:")
+  sensiblewrites.each do |sensiblewrite|
+    pid, ppid, time = Rbbt.load_lock(sensiblewrite + '.lock')
+    puts "  " << sensiblewrite + Log.color(:blue, " -- time: #{Time.now - time}; ppid: #{ppid}; pid: #{pid_msg pid}")
+  end
+  puts
+end
+
+jobs = Rbbt.jobs
+
+puts Log.color(:magenta, "# Workflows:")
+puts
+jobs.each do |workflow, tasks|
+  tasks.each do |task, jobs|
+    puts "* " << Log.color(:magenta, workflow) << "#" << Log.color(:yellow, task) << ": "
+    done = []
+    other = {}
+    jobs.each do |file,h|
+      status = h[:status]
+      if h[:status] == :done
+        done << file
+      else
+        other[status||"missing"] ||= []
+        other[status||"missing"] << [file, h[:pid]]
+      end
+    end
+    puts "  " << Log.color(:green, "done") << ": " << done.length.to_s
+    other.each do |status, list|
+      puts "  " << status_msg(status) << ": " << list.collect{|f,p| p.nil? ? f : f + " (#{pid_msg p})"} * ", "
+    end
+  end
+end
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.13.25
+  version: 5.13.26
 platform: ruby
 authors:
 - Miguel Vazquez
@@ -130,6 +130,7 @@ files:
 - lib/rbbt/entity.rb
 - lib/rbbt/fix_width_table.rb
 - lib/rbbt/knowledge_base.rb
+- lib/rbbt/monitor.rb
 - lib/rbbt/persist.rb
 - lib/rbbt/persist/tsv.rb
 - lib/rbbt/persist/tsv/cdb.rb
@@ -245,8 +246,10 @@ files:
 - share/rbbt_commands/stat/density
 - share/rbbt_commands/stat/log
 - share/rbbt_commands/study/task
+- share/rbbt_commands/system/clean
 - share/rbbt_commands/system/purge
 - share/rbbt_commands/system/report
+- share/rbbt_commands/system/status
 - share/rbbt_commands/tsv/attach
 - share/rbbt_commands/tsv/change_id
 - share/rbbt_commands/tsv/get