rbbt-util 5.5.59 → 5.5.60

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    Y2JmMzk4OWUzYjZkNTE2NTZkODI4NzQ3OGNmZTA3YTg0Y2NmMTMyMg==
+    NTllNjc1OWUxMDdmZmNhMzYyZmMyODU5YTczNzBmOWJkMDg1MTRlMQ==
   data.tar.gz: !binary |-
-    NDNlMTU0Y2JhMjcwMzlmYjkwMmY0NWU0OGJmODVlZDRmODdkZjVkZg==
+    ZTFhYjMxOTg0ZmUwMzBiZDRkMGRlYzU1YzcxNjMyOGMxOTQ1MDdjYg==
 SHA512:
   metadata.gz: !binary |-
-    YmI5ZGEyN2M4YjE0Mjg0YzVmMTM4NTk0NzhkZDJiODE1MTE5Zjk0NjQyM2Yy
-    MmY5MDIyNzRiMTJhMGI1OTM1MGFiZThmZDI1M2IyMDc2OGQ4OGRhMjVhNTEz
-    NWViMGU3MTlkMDA3N2QwNWVlNGEyNmQxNWM3ZWY4ZDA0ZmY1Yzc=
+    OWVkOWNhMzlmM2MzOGM5ZjI1MzEyZDZkMzViYjNjMTI0MDVhZDFkZjgzZDEx
+    ZGZmOWI2MDc0ZjQyMWU4ZGU2MmViNTU0ZWVjOWEzZDdhOGQyNDk2NTk4Yjcz
+    ZjJlZTkxZWYwYjk5MjhhMGM5MDBlZDk0ZmZkZmY3NDI0ZDA5NGM=
   data.tar.gz: !binary |-
-    NGNkMWM5OTFkY2ZiZTQyMDY1ZDhjZDFmMTM0YjQ2YTAwNGJhMTU5ZjM5ZWUy
-    YzBmMjM2ZjRjMTFkMTEwMDg0NmU2Mjk5MmY2NmFhNzMwMGNmNTUzOTljZmEx
-    ZjhkODIwN2Y4YzJiMDMxODVlZWZiZTIxMjcwOGZiMTM5MmU0MDQ=
+    NGRkZmJlZDg0ZmRlNTcxZDdjM2FlYTE0MTFkODc5NGUxZGJjMDIwNzczNjc1
+    ZjM5YjUxZTEzMGZiNmYxNjY4NGEyMWFiMTY4YjMzYmIwNGU1MzIwZDhlMDc4
+    MjVmN2Y3NzRiYTU2YzA0M2Q3ODFjN2JhNGM3Mzk2MzVhYWZlNTg=

data/etc/app.d/base.rb

@@ -25,6 +25,8 @@ use Rack::Session::Cookie, :key => 'rack.session',
 #{{{ DIRECTORIES
 local_var = Rbbt.var.find(:current)
 set :cache_dir           , local_var.sinatra.cache.find
+set :persist_dir         , local_var.sinatra.cache.persistence.find
+set :persist_options     , {:persist => true, :persist_dir => :persist_dir}
 set :file_dir            , local_var.sinatra.files.find
 set :permalink_dir       , local_var.sinatra.permalink.find
 set :favourites_dir      , local_var.sinatra.favourites.find

data/lib/rbbt/knowledge_base.rb

@@ -70,7 +70,6 @@ class KnowledgeBase
     @registry.keys
   end
 
-
   def description(name)
     @descriptions[name] ||= get_index(name).key_field.split("~")
   end
@@ -114,41 +113,45 @@ class KnowledgeBase
   end
  
   def get_database(name, options = {})
-    options = Misc.add_defaults options, :persist_dir => dir.databases
-    persist_options = Misc.pull_keys options, :persist
-
-    file, registered_options = registry[name]
-    options = open_options.merge(registered_options || {}).merge(options)
-    raise "Repo #{ name } not found and not registered" if file.nil?
-
-    code = [name, Misc.hash2md5(options)] * "_"
-    @databases[code] ||= begin 
-                           Log.low "Opening database #{ name } from #{ Misc.fingerprint file }. #{options}"
-                           Association.open(file, options, persist_options).
-                             tap{|tsv| tsv.namespace = self.namespace}
-                         end
+    @databases[Misc.fingerprint([name, options])] ||= \
+      begin 
+        Persist.memory("Database:" <<[name, self.dir] * "@") do
+          options = Misc.add_defaults options, :persist_dir => dir.databases
+          persist_options = Misc.pull_keys options, :persist
+
+          file, registered_options = registry[name]
+          options = open_options.merge(registered_options || {}).merge(options)
+          raise "Repo #{ name } not found and not registered" if file.nil?
+
+          Log.low "Opening database #{ name } from #{ Misc.fingerprint file }. #{options}"
+          Association.open(file, options, persist_options).
+            tap{|tsv| tsv.namespace = self.namespace}
+        end
+      end
   end
 
- 
-  def get_index(name, options = {})
-    options = Misc.add_defaults options, :persist_dir => dir.indices
-    persist_options = Misc.pull_keys options, :persist
-
-    file, registered_options = registry[name]
-    options = open_options.merge(registered_options || {}).merge(options)
-    raise "Repo #{ name } not found and not registered" if file.nil?
 
-    code = [name, Misc.hash2md5(options)] * "_"
-    @indices[code] ||= begin 
-                           Log.low "Opening index #{ name } from #{ Misc.fingerprint file }. #{options}"
-                           Association.index(file, options, persist_options).
-                             tap{|tsv| tsv.namespace = self.namespace}
-                         end
+  def get_index(name, options = {})
+    @indices[Misc.fingerprint([name, options])] ||= \
+      begin 
+        Persist.memory("Index:" <<[name, self.dir] * "@") do
+          options = Misc.add_defaults options, :persist_dir => dir.indices
+          persist_options = Misc.pull_keys options, :persist
+
+          file, registered_options = registry[name]
+          options = open_options.merge(registered_options || {}).merge(options)
+          raise "Repo #{ name } not found and not registered" if file.nil?
+
+          Log.low "Opening index #{ name } from #{ Misc.fingerprint file }. #{options}"
+          Association.index(file, options, persist_options).
+            tap{|tsv| tsv.namespace = self.namespace}
+        end
+      end
   end
 
-  def index(name, file, options = {}, persist_options = {})
-    @indices[name] = Association.index(file, open_options.merge(options), persist_options)
-  end
+  #def index(name, file, options = {}, persist_options = {})
+  #  @indices[name] = Association.index(file, open_options.merge(options), persist_options)
+  #end
 
   #{{{ Add manual database
   

data/lib/rbbt/persist.rb

@@ -54,9 +54,9 @@ module Persist
     prefix = Misc.process_options persist_options, :prefix
 
     if prefix.nil?
-      perfile = file.gsub(/\//, '>') 
+      perfile = file.to_s.gsub(/\//, '>') 
     else
-      perfile = prefix.to_s + ":" + file.gsub(/\//, '>') 
+      perfile = prefix.to_s + ":" + file.to_s.gsub(/\//, '>') 
     end
 
     if options.include? :filters
@@ -291,6 +291,14 @@ module Persist
       yield
     end
   end
+
+  def self.memory(name, options = {}, &block)
+    file = name
+    file << "_" << Misc.hash2md5(options) if options.any?
+    options = Misc.add_defaults options, :persist => true, :file => file
+
+    persist name, :memory, options, &block
+  end
 end
 
 module LocalPersist

data/lib/rbbt/util/cmd.rb

@@ -71,7 +71,7 @@ module CMD
 
       self.close if self.eof? and not self.closed?
 
-      data
+      data || ""
     end
   end
 

data/lib/rbbt/util/log.rb

@@ -27,6 +27,10 @@ module Log
 
   HIGHLIGHT = "\033[1m"
 
+  def self.color(severity)
+    SEVERITY_COLOR[severity]
+  end
+
   def self.log(message = nil, severity = MEDIUM, &block)
     return if severity < self.severity
     message ||= block.call if block_given?
@@ -35,7 +39,10 @@ module Log
     severity_color = SEVERITY_COLOR[severity]
     time = Time.now.strftime("%m/%d/%y-%H:%M:%S")
 
-    prefix = time << "[" << severity_color << severity.to_s << SEVERITY_COLOR[0] << "]"
+    sev_str = severity.to_s
+    sev_str = "" << HIGHLIGHT << sev_str << SEVERITY_COLOR[0] if severity >= INFO
+
+    prefix = time << "[" << severity_color << sev_str << SEVERITY_COLOR[0] << "]"
     message = "" << HIGHLIGHT << message << SEVERITY_COLOR[0] if severity >= INFO
     str = prefix << " " << message
 

data/lib/rbbt/util/misc.rb

@@ -240,6 +240,8 @@ module Misc
   end
 
   def self.fingerprint(obj)
+    #DEBUG
+    return "nil" if obj.nil?
     case obj
     when nil
       "nil"
@@ -251,6 +253,8 @@ module Misc
       else 
         "'" << obj << "'"
       end
+    when AnnotatedArray
+      "<E: #{fingerprint Annotated.purge(obj)} #{fingerprint obj.info}>"
     when Array
       if (length = obj.length) > 10
         "[#{length} --" <<  (obj.values_at(0,1, length / 2, -2, -1).collect{|e| fingerprint(e)} * ",") << "]"

data/lib/rbbt/workflow.rb

@@ -17,7 +17,7 @@ module Workflow
   end
   def self.extended(base)
     self.workflows << base
-    base.libdir = Path.caller_lib_dir.tap{|p| p.resource = base}
+    base.libdir = Path.setup(Path.caller_lib_dir).tap{|p| p.resource = base}
   end
 
   def self.require_remote_workflow(wf_name, url)

data/share/config.ru

@@ -1,41 +1,49 @@
 require 'rbbt'
 require 'rbbt/resource'
+$LOAD_PATH.unshift('lib') unless $LOAD_PATH.include?('lib')
 
-load Rbbt.etc['app.d/init.rb'].find
+def load_file(file)
+  if file.exists?
+    Log.info("Loading: " << file)
+    load file
+  end
+end
 
-$class_name = class_name = File.basename(FileUtils.pwd)
+def app_eval(app, file)
+  if file.exists?
+    app.class_eval do
+      Log.info("Loading: " << file)
+      eval file.read, nil, file
+    end
+  end
+end
 
-$app = app = eval "class #{class_name} < Sinatra::Base; self end"
+#{{{ INIT
+load_file Rbbt.etc['app.d/init.rb'].find
 
-$LOAD_PATH.unshift('lib')
+$class_name = class_name = File.basename(FileUtils.pwd)
+$app = app = eval "class #{class_name} < Sinatra::Base; self end"
 
 #{{{ PRE
-Log.info{"Loading: " << Rbbt.etc['app.d/pre.rb'].find if Rbbt.etc['app.d/pre.rb'].exists?}
-load Rbbt.etc['app.d/pre.rb'].find if Rbbt.etc['app.d/pre.rb'].exists?
+load_file Rbbt.etc['app.d/pre.rb'].find 
 
 #{{{ BASE
-app.class_eval do
-  Log.info{"Loading: " << Rbbt.etc['app.d/base.rb'].find}
-  eval Rbbt.etc['app.d/base.rb'].read, nil, Rbbt.etc['app.d/base.rb'].find
-end
+app_eval app, Rbbt.etc['app.d/base.rb'].find
 
 #{{{ RESOURCES
-Log.info{"Loading: " << Rbbt.etc['app.d/resources.rb'].find}
-load Rbbt.etc['app.d/resources.rb'].find
+load_file Rbbt.etc['app.d/resources.rb'].find
 
 #{{{ ENTITIES
-Log.info{"Loading: " << Rbbt.etc['app.d/entities.rb'].find}
-load Rbbt.etc['app.d/entities.rb'].find
+load_file Rbbt.etc['app.d/entities.rb'].find
 
 #{{{ FINDER
-app.class_eval do
-  Log.info{"Loading: " << Rbbt.etc['app.d/finder.rb'].find}
-  eval Rbbt.etc['app.d/finder.rb'].read
-end
+app_eval app, Rbbt.etc['app.d/finder.rb'].find
 
 #{{{ POST
-Log.info{"Loading: " << Rbbt.etc['app.d/post.rb'].find if Rbbt.etc['app.d/post.rb'].exists?}
-load Rbbt.etc['app.d/post.rb'].find if Rbbt.etc['app.d/post.rb'].exists?
+load_file Rbbt.etc['app.d/post.rb'].find if Rbbt.etc['app.d/post.rb'].exists?
+
+#{{{ PRELOAD
+load_file Rbbt.etc['app.d/preload.rb'].find if Rbbt.etc['app.d/preload.rb'].exists?
 
 #{{{ RUN
 $title = class_name

data/share/rbbt_commands/system/purge

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+
+options = SOPT.get("-a--app*")
+
+
+CMD.cmd("rbbt workflow monitor --quick -c -a -d #{Rbbt.var.jobs.find}").read
+CMD.cmd("rm -f  #{ Rbbt.tmp.tsv_open_locks.find.glob("*").collect{|f| "'#{f}'" } * " " }").read
+CMD.cmd("find #{Rbbt.share.find_all.collect{|f| "'#{f}'" } * " " } -name '*.lock' -delete").read 
+CMD.cmd("find #{Rbbt.var.find_all.collect{|f| "'#{f}'" } * " " } -name '*.lock' -delete").read 
+
+app_dir = Rbbt.etc.app_dir.exists? ? Rbbt.etc.app_dir.read.strip : Rbbt.apps.find
+Path.setup(app_dir)
+
+options[:app].split(/,|\s/).collect do |app|
+  d = app_dir[app].var.sinatra.cache.find 
+  CMD.cmd("rbbt workflow monitor --quick -c -a -d #{d}").read
+end if options[:app]

data/share/rbbt_commands/system/report

@@ -0,0 +1,67 @@
+#!/usr/bin/env ruby
+
+options = SOPT.get("-a--app*")
+
+workflow_dir = Rbbt.var.jobs.find
+
+def report_jobs(workflow_dir, title = "WORKFLOW")
+  error = {}
+  running = {}
+  dead = {}
+  CMD.cmd("rbbt workflow monitor --quick  -d '#{workflow_dir}'").read.split("\n").each do |line|
+    parts = line.split("/")
+    workflow, task, rest = parts[-3], parts[-2], parts[-1]
+    code = workflow.nil? ? nil : [workflow, task] * ":"
+    case
+    when rest =~ /error/
+      error[code] ||= 0
+      error[code] += 1
+    when rest =~ /dead/
+      dead[code] ||= 0
+      dead[code] += 1
+    when rest =~ /running/
+      running[code] ||= 0
+      running[code] += 1
+    when rest =~ /aborted/
+      dead[code] ||= 0
+      dead[code] += 1
+    else
+      raise rest
+    end
+  end
+
+  <<-EOF
+# #{title} JOBS
+Running jobs: #{ running.sort_by{|code,c| c}.reverse.collect{|code,c| [code, c].compact * " " } * ", " }
+Dead/aborted jobs: #{ dead.sort_by{|code,c| c}.reverse.collect{|code,c| [code, c].compact * " " } * ", " }#{Log.color(0)}
+Error jobs: #{ error.sort_by{|code,c| c}.reverse.collect{|code,c| [code, c].compact * " " } * ", " }#{Log.color(0)}
+  EOF
+end
+
+app_dir = Rbbt.etc.app_dir.exists? ? Rbbt.etc.app_dir.read.strip : Rbbt.apps.find
+Path.setup(app_dir)
+
+app_dirs = {}
+options[:app].split(/,|\s/).collect do |app|
+  d = app_dir[app].var.sinatra.cache.find 
+  report = report_jobs d, app
+  app_dirs[app] = report
+end if options[:app]
+
+puts <<EOF
+
+#{ report_jobs Rbbt.var.jobs.find}
+#{ app_dirs.collect{|d,report| report } * "\n" }
+
+# LOCKED TSV
+#{ Rbbt.var.tsv_open_locks.glob('*').collect{|f| "- " << File.basename(f) } * "\n" }
+
+# LOCKS
+#{ CMD.cmd("find #{Rbbt.share.find_all.collect{|f| "'#{f}'" } * " " } -name '*.lock'").read << CMD.cmd("find #{Rbbt.var.find_all.collect{|f| "'#{f}'" } * " " } -name '*.lock'").read }
+
+# PERSIST
+#{ CMD.cmd("find #{Rbbt.share.find_all.collect{|f| "'#{f}'" } * " " } -name '*.persist'").read << CMD.cmd("find #{Rbbt.var.find_all.collect{|f| "'#{f}'" } * " " } -name '*.persist'").read }
+EOF
+
+
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.5.59
+  version: 5.5.60
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-11-29 00:00:00.000000000 Z
+date: 2013-12-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -219,6 +219,8 @@ files:
 - share/rbbt_commands/resource/get
 - share/rbbt_commands/resource/produce
 - share/rbbt_commands/study/task
+- share/rbbt_commands/system/purge
+- share/rbbt_commands/system/report
 - share/rbbt_commands/tsv/attach
 - share/rbbt_commands/tsv/change_id
 - share/rbbt_commands/tsv/get
@@ -291,37 +293,37 @@ signing_key:
 specification_version: 4
 summary: Utilities for the Ruby Bioinformatics Toolkit (rbbt)
 test_files:
-- test/rbbt/tsv/test_accessor.rb
-- test/rbbt/tsv/test_index.rb
-- test/rbbt/tsv/test_util.rb
-- test/rbbt/tsv/test_filter.rb
-- test/rbbt/tsv/test_attach.rb
-- test/rbbt/tsv/test_manipulate.rb
-- test/rbbt/tsv/test_change_id.rb
-- test/rbbt/test_fix_width_table.rb
-- test/rbbt/association/test_index.rb
-- test/rbbt/association/test_item.rb
-- test/rbbt/test_association.rb
 - test/rbbt/test_workflow.rb
-- test/rbbt/workflow/test_step.rb
-- test/rbbt/workflow/test_task.rb
-- test/rbbt/workflow/test_soap.rb
 - test/rbbt/resource/test_path.rb
-- test/rbbt/test_tsv.rb
+- test/rbbt/util/test_cmd.rb
+- test/rbbt/util/test_chain_methods.rb
+- test/rbbt/util/test_simpleDSL.rb
+- test/rbbt/util/test_open.rb
+- test/rbbt/util/test_R.rb
+- test/rbbt/util/test_colorize.rb
+- test/rbbt/util/test_simpleopt.rb
+- test/rbbt/util/test_excel2tsv.rb
+- test/rbbt/util/test_filecache.rb
+- test/rbbt/util/test_misc.rb
+- test/rbbt/util/test_tmpfile.rb
+- test/rbbt/test_association.rb
 - test/rbbt/test_resource.rb
-- test/rbbt/test_annotations.rb
 - test/rbbt/test_entity.rb
 - test/rbbt/test_knowledge_base.rb
+- test/rbbt/association/test_index.rb
+- test/rbbt/association/test_item.rb
+- test/rbbt/test_tsv.rb
+- test/rbbt/workflow/test_soap.rb
+- test/rbbt/workflow/test_task.rb
+- test/rbbt/workflow/test_step.rb
 - test/rbbt/test_persist.rb
-- test/rbbt/util/test_filecache.rb
-- test/rbbt/util/test_tmpfile.rb
-- test/rbbt/util/test_excel2tsv.rb
-- test/rbbt/util/test_simpleopt.rb
-- test/rbbt/util/test_colorize.rb
-- test/rbbt/util/test_misc.rb
-- test/rbbt/util/test_open.rb
-- test/rbbt/util/test_cmd.rb
-- test/rbbt/util/test_chain_methods.rb
-- test/rbbt/util/test_R.rb
-- test/rbbt/util/test_simpleDSL.rb
+- test/rbbt/test_annotations.rb
+- test/rbbt/tsv/test_index.rb
+- test/rbbt/tsv/test_change_id.rb
+- test/rbbt/tsv/test_util.rb
+- test/rbbt/tsv/test_accessor.rb
+- test/rbbt/tsv/test_filter.rb
+- test/rbbt/tsv/test_attach.rb
+- test/rbbt/tsv/test_manipulate.rb
+- test/rbbt/test_fix_width_table.rb
 - test/test_helper.rb