rbbt-util 5.2.4 → 5.3.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    ZjIwN2RhYmRjNDg2MjMzMWNmYWU4NGQ5ZDUyYjcyNGU4MmExMzJhYw==
+    ZDNlMzk5N2E5ZWRkYWExZmVhNGYzZDliMjAwMDhkZWE5MDZmMjhjZA==
   data.tar.gz: !binary |-
-    NzAyMmUyMjg3ZWRmY2I1N2EwNDk1OGVlNTBmNWUyNmIyNTUxN2QwNg==
+    NGUzYzZiOTc2MGY2NmNhYWFmNGQxZjVjYWY3ZjgzOGFmZTJlN2RhMg==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    M2M0OTlkODVmMjY1ZWNjZWIzYjhjMTE2M2MyOGNiNWY5YjZiYjNiNDdkMTQx
-    MjkyNTk0MWRkZGM4ZjU3OTVjOTQ3ODg3YmNhNjVmZjljZDlmZGU3YTI4YWIw
-    MmFkZjcwMTE5OGFlNmI3MjRkZWI0NTQwZDBlZTg3ZTUwMWJhYjY=
+    ZmNiOTVmYWFlN2JhN2M0ZWU1YjA0M2VhZjM3OTgzZDA1OTAxZGUxYjMxNTg4
+    OGI3YjNjZTdiOTkzMGU5Yzc1YjM4YTliOWVhMGVmOGUwNWYwOTE1Nzk0MmRm
+    ZDJhYmVmZDAwZGJkNzRhNmJlNmI2MzZkODUwNTQwYTU4NTMyMTk=
   data.tar.gz: !binary |-
-    ZTIxNTBhZGQyZjg0ZjNiYjhlZjU3MTQ4M2YzMzgyNWNlZWE4MWMxNDg2OWUw
-    MzRlMzQyMzkyZWJiNDM1NjQ3ODljNmUwYjg1ZjJhYzYxOWE3MTZjZjc1ZDM3
-    OTI1ZjMzMTNmZDE4Mzc4OTQ3NDJmY2UyYTQzNWRjZDg0MjhlOWI=
+    OGIzZmFhYmQyMWYzY2U2YTFiMWExYzJkNGI3ZDY2YzE5OTFkMmViNzM5NjM3
+    YWViMjBiOTVjM2I1M2Y3MDRmYjFhZWFlMjIyM2FkNzI1NjFmMzc0MGMyODBk
+    YmVmMGJiMWQyYzQzYTkwZWIwYjcyZmJiZTkzODYyNmI5YjZhM2M=

data/bin/rbbt

@@ -3,7 +3,7 @@
 require 'rbbt'
 require 'rbbt/util/simpleopt'
 
-options = SOPT.get("--log* Log level from 0 (debug) 7 (errors):--command_dir* Directory from where to load commands")
+options = SOPT.get("--log* Log level from 0 (debug) 7 (errors):--command_dir* Directory from where to load commands:--profile Profile execution")
 
 if options[:log]
   Log.severity = options[:log].to_i
@@ -15,7 +15,12 @@ else
   $rbbt_command_dir = Rbbt.share.rbbt_commands
 end
 
-SOPT.synopsys = "rbbt <command> <subcommand> ... -a --arg1 --arg2='value'"
+if options[:profile]
+  require 'ruby-prof'
+  RubyProf.start
+end
+
+SOPT.synopsys = "rbbt <command> <subcommand> ... -a --arg1 --arg2='value' --arg3 'another-value'"
 SOPT.summary = "Ruby bioinformatics toolkit"
 SOPT.description = <<-EOF
 This command controls many aspects of the Rbbt framework, from configuration tasks to running applications. 
@@ -41,20 +46,28 @@ end
 dir = $rbbt_command_dir
 prev = []
 
-while ARGV.any?
+begin
+  while ARGV.any?
     command = ARGV.shift
     case
     when File.directory?(dir[command].find)
-        prev << command
-        dir = dir[command]
+      prev << command
+      dir = dir[command]
     when dir[command].exists?
-        load dir[command].find
-        exit 0
+      load dir[command].find
+      exit 0
     else
-        puts "Error: Command not understood"
-        puts
-        break
+      puts "Error: Command not understood"
+      puts
+      break
     end
+  end
+ensure
+  if options[:profile]
+    result = RubyProf.stop
+    printer = RubyProf::FlatPrinter.new(result)
+    printer.print(STDOUT, :min_percent => 10)
+  end
 end
 
 puts SOPT.doc

data/bin/rbbt_monitor.rb

@@ -26,7 +26,7 @@ def print_job(file, info, severity_color = nil)
   else
     info ||= {:status => :missing_info_file}
     str = [clean_file, info[:status].to_s] * " [ STATUS = " + " ]" 
-    str += " (#{running?(info)? :running : :dead} #{info[:pid]})" if info.include? :pid
+    str += " (#{running?(info)? :running : :dead} #{info[:pid]})" if info[:pid]
     str += " (children: #{info[:children_pids].collect{|pid| [pid, Misc.pid_exists?(pid) ? "R" : "D"] * ":"} * ", "})" if info.include? :children_pids
 
     str = "#{severity_color}" <<  str  << "\033[0m" if severity_color
@@ -42,8 +42,8 @@ def list_jobs(options)
     clean_file = file.sub('.info','')
     begin
       next if File.exists? clean_file and $quick
-      info = YAML.load(Open.read(file))
-      next if File.exists? clean_file and not running? info
+      info = Step::INFO_SERIALIAZER.load(Open.read(file, :mode => 'rb'))
+      next if (File.exists?(clean_file) or info[:status] == :done) and (info[:children_pids].nil? or info[:children_done] or info[:children_pids].select{|pid| Misc.pid_exists? pid}.empty?)
     rescue Exception
       puts "Error parsing info file: #{ file }"
       info = nil
@@ -82,13 +82,13 @@ end
 def clean_jobs(options)
   info_files.each do |file|
     clean_file = file.sub('.info','')
-    next if File.exists? clean_file
     info = nil
+    next if File.exists? clean_file
     begin
-      info = YAML.load(Open.read(file))
-    rescue
+      info = Step::INFO_SERIALIAZER.load(Open.read(file, :mode => 'rb'))
+    rescue Exception
       Log.debug "Error process #{ file }"
-      raise $!
+      remove_job file if options[:errors]
     end
     case
     when options[:all]
@@ -115,7 +115,7 @@ when (options[:clean] and not options[:list])
   end
 else
   if options[:file]
-    info = YAML.load(Open.read(options[:file]))
+    info = Step::INFO_SERIALIAZER.load(Open.read(options[:file], :mode => 'rb'))
     print_job options[:file], info
   else
     list_jobs options

data/lib/rbbt/annotations.rb

@@ -106,8 +106,10 @@ module Annotated
 
   def annotate(object)
 
+    return object unless Array === object or String === object
+
     annotation_types.each do |annotation|
-      object.extend annotation
+      object.extend annotation unless annotation === object
     end
 
     object.instance_variable_set(:@annotation_types, nil)
@@ -145,6 +147,25 @@ module Annotated
     end
   end
 
+  def self.purge(object)
+    case object
+    when String
+      object.respond_to?(:clean_annotations) ?
+        object.clean_annotations : 
+        object
+    when Array
+      object.respond_to?(:clean_annotations) ?
+        object.clean_annotations : 
+        object.collect{|e| Annotated.purge e}
+    when Hash
+      object.each do |key, value|
+        object[key] = Annotated.purge value
+      end
+    else
+      object
+    end
+  end
+
   def make_list
     new = [self]
     self.annotate(new)

data/lib/rbbt/annotations/util.rb

@@ -14,7 +14,7 @@ module Annotated
 
     annotation_types.each do |mod|
       mod = Misc.string2const(mod) if String === mod
-      object.extend mod
+      object.extend mod unless mod === object
     end
 
     object.instance_variable_set(:@annotation_values, info) 

data/lib/rbbt/entity.rb

@@ -0,0 +1,162 @@
+require 'rbbt/annotations'
+
+module Entity 
+
+  class << self
+    attr_accessor :formats, :entity_property_cache
+  end
+
+  self.entity_property_cache = "var/entity_property"
+  self.formats = {}
+  
+  UNPERSISTED_PREFIX = "entity_unpersisted_property_"
+
+
+  def self.extended(base)
+    base.extend Annotation
+    Entity.formats[base.to_s] = base
+
+    base.module_eval do
+      attr_accessor :_ary_property_cache
+
+      attr_accessor :template, :list_template, :action_template, :list_action_template, :keep_id
+
+      def _ary_property_cache
+        @_ary_property_cache ||= {}
+      end
+
+      def self.format=(formats)
+        formats = [formats] unless Array === formats
+        formats.each do |format|
+          Entity.formats[format] = self
+        end
+      end
+
+      def property(*args, &block)
+        class << self; self; end.property(*args,&block)
+      end
+
+      def to_yaml(*args)
+        self.clean_annotations.dup.to_yaml(*args)
+      end
+
+
+      def encode_with(coder)
+        coder.scalar = clean_annotations
+      end
+
+      def marshal_dump
+        clean_annotations
+      end
+
+      def consolidate
+        self.inject(nil){|acc,e| 
+          if acc.nil?
+            acc = e
+          else
+            acc.concat e
+          end
+        }
+      end
+
+      def self.property(name, &block)
+        case
+        when (Hash === name and name.size == 1)
+          name, type = name.collect.first
+        when (String === name or Symbol === name)
+          type = :single
+        else
+          raise "Format of name ( => type) not understood: #{name.inspect}"
+        end
+
+        name = name.to_s unless String === name
+
+        persisted_name = UNPERSISTED_PREFIX + name
+        self.remove_method persisted_name if methods.include? persisted_name
+
+        case type
+        when :both
+          define_method name, &block 
+ 
+        when :single, :single2array
+          single_name = "_single_" << name
+          define_method single_name, &block 
+          define_method name do |*args|
+            if Array === self
+              self.collect{|e| e.send(name, *args)}
+            else
+              self.send(single_name, *args)
+            end
+          end
+        when :array, :array2single
+          ary_name = "_ary_" << name
+          define_method ary_name, &block 
+          
+          define_method name do |*args|
+            #ary_name = "_ary_" << __method__.to_s
+            case
+            when Array === self
+              self.send(ary_name, *args)
+            when (Array === self.container and not self.container_index.nil? and self.container.respond_to? ary_name)
+              cache_code = Misc.hash2md5({:name => ary_name, :args => args})
+              res = (self.container._ary_property_cache[cache_code] ||=  self.container.send(name, *args))
+              if Hash === res
+                res[self]
+              else
+                res[self.container_index]
+              end
+            else
+              res = self.make_list.send(ary_name, *args)
+              Hash === res ? res[self] : res[0]
+            end
+          end
+
+        end
+      end
+
+      def self.persist(method_name, type = nil, options = {})
+        type = :memory if type.nil?
+        options ||= {}
+        options = Misc.add_defaults options, :dir => Entity.entity_property_cache
+
+        orig_name = UNPERSISTED_PREFIX + method_name.to_s
+        alias_method orig_name, method_name unless self.instance_methods.include? orig_name.to_sym
+
+        define_method method_name do |*args|
+          id = self.id
+          persist_name = __method__.to_s << ":" << (Array === id ? Misc.hash2md5(:id => id) : id)
+          persist_name << ":" << Misc.hash2md5({:args => args}) unless args.nil? or args.empty?
+
+          persist_options = options
+          persist_options = persist_options.merge(:other => {:args => args}) if args.any?
+
+          Persist.persist(persist_name, type, persist_options) do
+            self.send(orig_name, *args)
+          end
+        end
+      end
+
+      def self.unpersist(method_name)
+        return unless persisted? method_name
+        orig_name = UNPERSISTED_PREFIX + method_name.to_s
+
+        alias_method method_name, orig_name
+        remove_method orig_name
+      end
+
+      def self.persisted?(method_name)
+        orig_name = UNPERSISTED_PREFIX + method_name.to_s
+        instance_methods.include? orig_name.to_sym
+      end
+
+      def self.with_persisted(method_name)
+        persisted = persisted? method_name
+        persist method_name unless persisted
+        res = yield
+        unpersist method_name unless persisted
+        res
+      end
+
+    end 
+  end
+end

data/lib/rbbt/fix_width_table.rb

@@ -244,4 +244,11 @@ class FixWidthTable
     }
   end
 
+  def chunked_values_at(keys, max = 5000)
+    Misc.ordered_divide(keys, max).inject([]) do |acc,c|
+      new = self.values_at(*c)
+      new.annotate acc if new.respond_to? :annotate and acc.empty?
+      acc.concat(new)
+    end
+  end
 end

data/lib/rbbt/persist.rb

@@ -17,15 +17,16 @@ module Persist
   end
 
   MEMORY = {} unless defined? MEMORY
+  MAX_FILE_LENGTH = 150
 
   def self.newer?(path, file)
-    return true if not File.exists? file
+    return true if not Open.exists? file
     return true if File.mtime(path) < File.mtime(file)
     return false
   end
 
   def self.is_persisted?(path, persist_options = {})
-    return false if not File.exists? path
+    return false if not Open.exists? path
     return false if TrueClass === persist_options[:update]
 
     check = persist_options[:check]
@@ -66,6 +67,8 @@ module Persist
     clean_options.delete :unnamed
     clean_options.delete "unnamed"
 
+    filename = filename[0..MAX_FILE_LENGTH] << Misc.digest(filename[MAX_FILE_LENGTH+1..-1]) if filename.length > MAX_FILE_LENGTH + 10
+
     options_md5 = Misc.hash2md5 clean_options
     filename  << ":" << options_md5 unless options_md5.empty?
 
@@ -248,6 +251,7 @@ module Persist
         end
 
       else
+
         if is_persisted?(path, persist_options)
           Log.low "Persist up-to-date: #{ path } - #{persist_options.inspect[0..100]}"
           return nil if persist_options[:no_load]
@@ -255,6 +259,7 @@ module Persist
         else
           Log.medium "Persist create: #{ path } - #{persist_options.inspect[0..100]}"
         end
+
         begin
           res = yield
           Misc.lock(path) do
@@ -262,8 +267,8 @@ module Persist
           end
           res
         rescue
-          Log.high "Error in persist. #{File.exists?(path) ? "Erasing '#{ path }'" : ""}"
-          FileUtils.rm path if File.exists? path 
+          Log.high "Error in persist. #{Open.exists?(path) ? "Erasing '#{ path }'" : ""}"
+          FileUtils.rm path if Open.exists? path 
           raise $!
         end
       end
@@ -279,16 +284,18 @@ module LocalPersist
   attr_accessor :local_persist_dir
 
   def local_persist_dir
-    @local_persist_dir ||= Rbbt.var.cache.persistence.find(:lib) if defined? Rbbt
+    @local_persist_dir ||= Rbbt.var.cache.persistence if defined? Rbbt
     @local_persist_dir
   end
 
-  def local_persist(name, type = nil, options= {}, &block)
-    Persist.persist(name, type, options.merge({:dir => local_persist_dir}), &block)
+  def local_persist(name, type = nil, options= {}, persist_options = nil, &block)
+    persist_options ||= {}
+    persist_options = {:dir => local_persist_dir}.merge persist_options
+    persist_options[:other] = options
+    Persist.persist(name, type, persist_options, &block)
   end
 
   def local_persist_tsv(source, name, opt = {}, options= {}, &block)
-    Persist.persist_tsv(source, name, opt, options.merge({:dir => local_persist_dir}), &block)
+    Persist.persist_tsv(source, name, opt, options.merge({:dir => local_persist_dir, :persist => true}), &block)
   end
-
 end

data/lib/rbbt/persist/tsv.rb

@@ -25,6 +25,10 @@ module Persist
       class << database
         attr_accessor :writable, :closed, :persistence_path, :tokyocabinet_class
 
+        def prefix(key)
+          range(key, 1, key + 255.chr, 1)
+        end
+
         def closed?
           @closed
         end
@@ -82,7 +86,7 @@ module Persist
         end
 
         def write_and_close
-          lock_filename = Persist.persistence_path(persistence_path, {:dir => Rbbt.tmp.tsv_open_locks.find})
+          lock_filename = Persist.persistence_path(persistence_path, {:dir => TSV.lock_dir})
           Misc.lock(lock_filename) do
             write if @closed or not write?
             res = begin
@@ -124,8 +128,11 @@ module Persist
     database.persistence_path ||= path
     database.tokyocabinet_class = tokyocabinet_class
 
-    TSV.setup database
-    database.serializer = serializer || database.serializer 
+    unless serializer == :clean
+      TSV.setup database
+      database.serializer = serializer || database.serializer
+      database.fields
+    end
 
     database
   end
@@ -153,18 +160,18 @@ module Persist
 
              path = persistence_path(filename, persist_options, options)
 
-             if is_persisted? path
+             if is_persisted? path and not persist_options[:update]
                Log.debug "TSV persistence up-to-date: #{ path }"
                lock_filename = Persist.persistence_path(path, {:dir => Rbbt.tmp.tsv_open_locks.find})
-               return Misc.lock(lock_filename) do open_tokyocabinet(path, false); end
+               return Misc.lock(lock_filename) do open_tokyocabinet(path, false, nil, persist_options[:engine] || TokyoCabinet::HDB); end
              else
                Log.medium "TSV persistence creating: #{ path }"
              end
 
              FileUtils.rm path if File.exists? path
 
-             data = open_tokyocabinet(path, true, persist_options[:serializer])
-             data.serializer = :type unless data.serializer
+             data = open_tokyocabinet(path, true, persist_options[:serializer], persist_options[:engine] || TokyoCabinet::HDB)
+             data.serializer = :type if TSV === data and data.serializer.nil?
 
              data.close
 
@@ -197,5 +204,4 @@ module Persist
 
     data
   end
-
 end