rbbt-util 5.13.31 → 5.13.32

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 22e6c0ab1705d1790142288b6bc9943665830554
-  data.tar.gz: 7873db2daa7dafc6ee9b8e1ba9ac12191c8c5d8b
+  metadata.gz: 03d6c0cd25cb674bc5fb28731e6c936d32a8468c
+  data.tar.gz: 0a8c5ba0281f806e9d8712e527b011bae576bacf
 SHA512:
-  metadata.gz: 07dee96dd2fabee74fbf1ae2852a657f16f30f16c818c092ebd352a6769b080155881a255a48098ebcf749efa95abb13cd9e9da82cecdbdec1a0e415be0c1940
-  data.tar.gz: 547fd0998e00974bc5ecd8bc3ba1b8a98db556cfb48d8c2e2fb0e327eeaaa9af6023112d3ebfdde613911cc459d54c23c60221dc5f55ce6ac40e288d7b33a027
+  metadata.gz: 24555145911d44367c1b9ddfe30c96b7e205811b9838508e68faebeb07da384b4fdc1d9bed59a221f97b00e51ba8ea816fccbb7c26517de79433e3306a9e8fbc
+  data.tar.gz: 6b275e80ce80ac94a8067c8f255c50d718fab0717934ff46e5354640951d4f94ae5361cb92d429b2cd14ba3edaea35f5dd26e9748eb4021a53bd1c33f576592b

data/bin/rbbt

@@ -42,8 +42,19 @@ $ rbbt <command> <subcommand> ... -a --arg1 --arg2='value' --arg3 'another-value
 --nocolor               #{Log.color :yellow, "Disable colored output"}
 --nobar                 #{Log.color :yellow, "Disable progress report"}
 --locate_file           #{Log.color :yellow, "Report the location of the script instead of executing it"}
+--dump_mem*             #{Log.color :yellow, "Dump strings in memory each second into file"}
+--no_lock_id           #{Log.color :yellow, "Do not track lockfiles with ids (for high-througput and high-concurrency"}
 EOF
 
+if options.delete(:no_lock_id)
+  Misc.use_lock_id = false
+end
+
+if mem_dump = options.delete(:dump_mem)
+  require 'rbbt/monitor'
+  Rbbt.dump_memory(mem_dump, Symbol)
+end
+
 if options.delete :nobar
   ENV["RBBT_NO_PROGRESS"] = "true"
 end

data/lib/rbbt/monitor.rb

@@ -5,28 +5,170 @@ module Rbbt
   LOCK_DIRS = Rbbt.share.find_all  + Rbbt.var.cache.persistence.find_all +  Rbbt.var.jobs.find_all +
     Rbbt.tmp.tsv_open_locks.find_all + Rbbt.tmp.persist_locks.find_all 
 
+  SENSIBLE_WRITE_DIRS = Misc.sensiblewrite_dir.find_all
+
+  PERSIST_DIRS    = Rbbt.share.find_all  + Rbbt.var.cache.persistence.find_all  
+
+  JOB_DIRS = Rbbt.var.jobs.find_all
+
+  def self.dump_memory(file, obj = nil)
+    Log.info "Dumping #{obj} objects into #{ file }"
+    Thread.new do
+      while true
+        Open.write(file) do |f|
+          Thread.exclusive do
+            GC.start
+            ObjectSpace.each_object(obj) do |o|
+              f.puts "---"
+              f.puts(String === o ? o : o.inspect)
+            end
+          end
+        end
+        FileUtils.cp file, file + '.save'
+        sleep 3
+      end
+    end
+  end
+
+  def self.file_time(file)
+    ctime = File.ctime file
+    atime = File.atime file
+    elapsed = Time.now - ctime
+    {:ctime => ctime, :atime => atime, :elapsed => elapsed}
+  end
+
+  #{{{ LOCKS
+
   def self.locks(dirs = LOCK_DIRS)
     dirs.collect do |dir|
-      dir.glob("**/*.lock")
-    end.flatten
+      next unless Open.exists? dir
+      `find "#{ dir }" -name "*.lock"`.split "\n"
+    end.compact.flatten
   end
 
-  SENSIBLE_WRITE_DIRS = Misc.sensiblewrite_dir.find_all
+  def self.lock_info(dirs = LOCK_DIRS)
+    lock_info = {}
+    locks(dirs).each do |f|
+      begin
+        i = file_time(f)
+        if File.size(f) > 0 
+          info = Open.open(f) do |s|
+            YAML.load(s) 
+          end
+          i[:pid] = info[:pid]
+          i[:ppid] = info[:ppid]
+        end
+        lock_info[f] = i
+      rescue
+        Log.exception $!
+      end
+    end
+    lock_info
+  end
+
+  #{{{ SENSIBLE WRITES
+
   def self.sensiblewrites(dirs = SENSIBLE_WRITE_DIRS)
     dirs.collect do |dir|
-      dir.glob("**/*").reject{|f| f =~ /\.lock$/ }
-    end.flatten
+      next unless Open.exists? dir
+      `find "#{ dir }" -not -name "*.lock" -not -type d`.split "\n"
+    end.compact.flatten
   end
 
-  PERSIST_DIRS    = Rbbt.share.find_all  + Rbbt.var.cache.persistence.find_all  
+  def self.sensiblewrite_info(dirs = SENSIBLE_WRITE_DIRS)
+    info = {}
+    sensiblewrites(dirs).each do |f|
+      begin
+        i = file_time(f)
+        info[f] = i
+      rescue
+        Log.exception $!
+      end
+    end
+    info
+  end
+
+  # PERSISTS
+  
   def self.persists(dirs = PERSIST_DIRS)
     dirs.collect do |dir|
-      dir.glob("**/*.persist").reject{|f| f =~ /\.lock$/ }
-    end.flatten
+      next unless Open.exists? dir
+      `find "#{ dir }" -name "*.persist"`.split "\n"
+    end.compact.flatten
   end
 
-  JOB_DIRS = Rbbt.var.jobs.find_all
-  def self.jobs(dirs = JOB_DIRS)
+  def self.persist_info(dirs = PERSIST_DIRS)
+    info = {}
+    persists(dirs).each do |f|
+      begin
+        i = file_time(f)
+        info[f] = i
+      rescue
+        Log.exception $!
+      end
+    end
+    info
+  end
+
+  # PERSISTS
+
+  def self.job_info(workflows = nil, tasks = nil, dirs = JOB_DIRS)
+    require 'rbbt/workflow/step'
+
+    workflows = [workflows] if workflows and not Array === workflows
+    workflows = workflows.collect{|w| w.to_s} if workflows
+
+    tasks = [tasks] if tasks and not Array === tasks
+    tasks = tasks.collect{|w| w.to_s} if tasks
+
+    jobs = {}
+    dirs.collect do |dir|
+      next unless Open.exists? dir
+
+      dir.glob("*").collect do |workflowdir|
+        workflow = File.basename(workflowdir)
+        next if workflows and not workflows.include? workflow
+
+        workflowdir.glob("*").collect do |taskdir|
+          task = File.basename(taskdir)
+          next if tasks and not tasks.include? task
+
+          files = `find "#{ taskdir }/" -not -type d -not -path "*/*.files/*"`.split("\n").sort
+          _files = Set.new files
+          TSV.traverse files, :type => :array, :into => jobs do |file|
+            if m = file.match(/(.*).info$/)
+              file = m[1]
+            end
+
+            name = file[taskdir.length+1..-1]
+            info_file = file + '.info'
+
+            info = {}
+
+            info[:workflow] = workflow
+            info[:task] = task
+            info[:name] = name
+
+            if _files.include? file
+              info = info.merge(file_time(file))
+              info[:done] = true
+              info[:info_file] = File.exists?(info_file) ? info_file : nil
+            else
+              info = info.merge({:info_file => info_file, :done => false})
+            end
+
+            [file, info]
+          end
+
+        end.compact.flatten
+      end.compact.flatten
+    end.compact.flatten
+    jobs
+  end
+
+  # REST
+
+  def self.__jobs(dirs = JOB_DIRS)
     job_files = {}
     dirs.each do |dir|
       workflow_dirs = dir.glob("*").each do |wdir|

data/lib/rbbt/persist/tsv/adapter.rb

@@ -0,0 +1,109 @@
+require 'rbbt/tsv'
+module Persist
+  module TSVAdapter
+    attr_accessor :persistence_path, :closed, :writable, :mutex
+
+    MAX_CHAR = 255.chr
+
+    def mutex
+      @mutex ||= Mutex.new
+    end
+
+    def prefix(key)
+      range(key, 1, key + MAX_CHAR, 1)
+    end
+
+    def get_prefix(key)
+      keys = prefix(key)
+      select(:key => keys)
+    end
+
+    def closed?
+      @closed
+    end
+
+    def close
+      @closed = true
+      super
+      self
+    end
+
+    def write?
+      @writable
+    end
+
+    def read?
+      ! write?
+    end
+
+    def collect
+      res = []
+      each do |key, value|
+        res << if block_given?
+                 yield key, value
+        else
+          [key, value]
+        end
+      end
+      res
+    end
+
+    def delete(key)
+      out(key)
+    end
+
+    def write_and_read
+      lock_filename = Persist.persistence_path(persistence_path + '.write', {:dir => TSV.lock_dir})
+      Misc.lock(lock_filename) do
+        mutex.synchronize do
+          write if closed? or not write?
+          res = begin
+                  yield
+                ensure
+                  read
+                end
+          res
+        end
+      end
+    end
+
+    def write_and_close
+      lock_filename = Persist.persistence_path(persistence_path + '.write', {:dir => TSV.lock_dir})
+      Misc.lock(lock_filename) do
+        mutex.synchronize do
+          write if closed? or not write?
+          res = begin
+                  yield
+                ensure
+                  close
+                end
+          res
+        end
+      end
+    end
+
+    def read_and_close
+      mutex.synchronize do
+        read if closed? or not read?
+        res = begin
+                yield
+              ensure
+                close
+              end
+        res
+      end
+    end
+
+
+    def merge!(hash)
+      hash.each do |key,values|
+        self[key] = values
+      end
+    end
+
+
+    def range(*args)
+      super(*args) #- TSV::ENTRY_KEYS.to_a
+    end
+  end
+end

data/lib/rbbt/persist/tsv/kyotocabinet.rb

@@ -3,7 +3,8 @@ require 'kyotocabinet'
 module Persist
 
   module KCAdapter
-    attr_accessor :persistence_path, :kyotocabinet_class, :closed, :writable
+    include Persist::TSVAdapter
+    attr_accessor :kyotocabinet_class
 
     def self.open(path, write, kyotocabinet_class = "kch")
       real_path = path + ".#{kyotocabinet_class}"
@@ -24,33 +25,10 @@ module Persist
       database
     end
 
-    def keys
-      keys = []
-      each_key{|k| keys.concat k}
-      keys
-    end
-
-    def prefix(key)
-      range(key, 1, key + 255.chr, 1)
-    end
-
-    def get_prefix(key)
-      keys = prefix(key)
-      select(:key => keys)
-    end
-
-    def include?(key)
-      value = get(key)
-      ! value.nil?
-    end
-
-    def closed?
-      @closed
-    end
-
     def close
       @closed = true
       super
+      self
     end
 
     def read(force = false)
@@ -96,58 +74,6 @@ module Persist
     def delete(key)
       out(key)
     end
-
-    def write_and_read
-      lock_filename = Persist.persistence_path(persistence_path, {:dir => TSV.lock_dir})
-      Misc.lock(lock_filename) do
-        write if @closed or not write?
-        res = begin
-                yield
-              ensure
-                read
-              end
-        res
-      end
-    end
-
-    def write_and_close
-      lock_filename = Persist.persistence_path(persistence_path, {:dir => TSV.lock_dir})
-      Misc.lock(lock_filename) do
-        write if @closed or not write?
-        res = begin
-                yield
-              ensure
-                close
-              end
-        res
-      end
-    end
-
-    def read_and_close
-      read if @closed or write?
-      res = begin
-              yield
-            ensure
-              close
-            end
-      res
-    end
-
-    def merge!(hash)
-      hash.each do |key,values|
-        self[key] = values
-      end
-    end
-
-    #def []=(key,value)
-    #  super(key,value)
-    #  self.synchronize
-    #end
-
-
-    def range(*args)
-      super(*args) - TSV::ENTRY_KEYS.to_a
-    end
   end
 
 

data/lib/rbbt/persist/tsv/lmdb.rb

@@ -3,8 +3,7 @@ require 'lmdb'
 module Persist
 
   module LMDBAdapter
-    attr_accessor :persistence_path, :closed
-
+    include Persist::TSVAdapter
     def self.open(path, write)
 
       database = CONNECTIONS[path] ||= begin
@@ -21,40 +20,6 @@ module Persist
       database
     end
 
-    def keys
-      keys = []
-      cursor do |cursor|
-        while p = cursor.next
-          keys << p.first
-        end
-      end
-      keys
-    end
-    
-    def include?(key)
-      self.send(:[], key, true)
-    end
-
-    def closed?
-      false
-    end
-
-    def close
-      self
-    end
-
-    def read(force = false)
-      self
-    end
-
-    def write(force = true)
-      self
-    end
-
-    def write?
-      @writable
-    end
-
     def each
       cursor do |cursor|
         while pair = cursor.next
@@ -77,60 +42,8 @@ module Persist
       end
       res
     end
-
-    def delete(key)
-      out(key)
-    end
-
-    def write_and_read
-      lock_filename = Persist.persistence_path(persistence_path, {:dir => TSV.lock_dir})
-      Misc.lock(lock_filename) do
-        write if @closed or not write?
-        res = begin
-                yield
-              ensure
-                read
-              end
-        res
-      end
-    end
-
-    def write_and_close
-      lock_filename = Persist.persistence_path(persistence_path, {:dir => TSV.lock_dir})
-      Misc.lock(lock_filename) do
-        write if @closed or not write?
-        res = begin
-                yield
-              ensure
-                close
-              end
-        res
-      end
-    end
-
-    def read_and_close
-      read if @closed or write?
-      res = begin
-              yield
-            ensure
-              close
-            end
-      res
-    end
-
-    def merge!(hash)
-      hash.each do |key,values|
-        self[key] = values
-      end
-    end
-
-
-    def range(*args)
-      super(*args) - TSV::ENTRY_KEYS.to_a
-    end
   end
 
-
   def self.open_lmdb(path, write, serializer = nil)
     write = true unless File.exists? path