rbbt-util 5.11.4 → 5.11.5

data/lib/rbbt/util/misc/concurrent_stream.rb

@@ -0,0 +1,69 @@
+module ConcurrentStream
+  attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined
+
+  def joined?
+    @joined
+  end
+
+  def join
+
+    if @threads and @threads.any?
+      @threads.each do |t| 
+        t.join unless t == Thread.current
+      end
+      @threads = []
+    end
+
+    if @pids and @pids.any?
+      @pids.each do |pid| 
+        begin
+          Process.waitpid(pid, Process::WUNTRACED)
+          raise "Error joining process #{pid} in #{self.inspect}" unless $?.success?
+        rescue Errno::ECHILD
+        end
+      end 
+      @pids = []
+    end
+
+    if @callback and not joined?
+      @callback.call
+      @callback = nil
+    end
+
+    @joined = true
+  end
+
+  def abort
+    @threads.each{|t| t.raise Aborted.new unless t = Thread.current } if @threads
+    @threads.each{|t| t.join unless t = Thread.current } if @threads
+    @pids.each{|pid| Process.kill :INT, pid } if @pids
+    @pids.each{|pid| Process.waitpid pid } if @pids
+    @abort_callback.call if @abort_callback
+    @abort_callback = nil
+  end
+
+  def self.setup(stream, options = {}, &block)
+    threads, pids, callback, filename = Misc.process_options options, :threads, :pids, :callback, :filename
+    stream.extend ConcurrentStream unless ConcurrentStream === stream
+
+    stream.threads ||= []
+    stream.pids ||= []
+    stream.threads.concat(Array === threads ? threads : [threads]) unless threads.nil? 
+    stream.pids.concat(Array === pids ? pids : [pids]) unless pids.nil? or pids.empty?
+
+    callback = block if block_given?
+    if stream.callback and callback
+      old_callback = stream.callback
+      stream.callback = Proc.new do
+        old_callback.call
+        callback.call
+      end
+    else
+      stream.callback = callback
+    end
+
+    stream.filename = filename unless filename.nil?
+
+    stream
+  end
+end

data/lib/rbbt/util/misc/development.rb

@@ -0,0 +1,95 @@
+module Misc
+  def self.benchmark(repeats = 1, message = nil)
+    require 'benchmark'
+    res = nil
+    begin
+      measure = Benchmark.measure do
+        repeats.times do
+          res = yield
+        end
+      end
+      if message
+        puts "#{message }: #{ repeats } repeats"
+      else
+        puts "Benchmark for #{ repeats } repeats"
+      end
+      puts measure
+    rescue Exception
+      puts "Benchmark aborted"
+      raise $!
+    end
+    res
+  end
+
+  def self.profile_html(options = {})
+    require 'ruby-prof'
+    RubyProf.start
+    begin
+      res = yield
+    rescue Exception
+      puts "Profiling aborted"
+      raise $!
+    ensure
+      result = RubyProf.stop
+      printer = RubyProf::MultiPrinter.new(result)
+      TmpFile.with_file do |dir|
+        FileUtils.mkdir_p dir unless File.exists? dir
+        printer.print(:path => dir, :profile => 'profile')
+        CMD.cmd("firefox  -no-remote  '#{ dir }'")
+      end
+    end
+
+    res
+  end
+
+  def self.profile_graph(options = {})
+    require 'ruby-prof'
+    RubyProf.start
+    begin
+      res = yield
+    rescue Exception
+      puts "Profiling aborted"
+      raise $!
+    ensure
+      result = RubyProf.stop
+      #result.eliminate_methods!([/annotated_array_clean_/])
+      printer = RubyProf::GraphPrinter.new(result)
+      printer.print(STDOUT, options)
+    end
+
+    res
+  end
+
+  def self.profile(options = {})
+    require 'ruby-prof'
+    RubyProf.start
+    begin
+      res = yield
+    rescue Exception
+      puts "Profiling aborted"
+      raise $!
+    ensure
+      result = RubyProf.stop
+      printer = RubyProf::FlatPrinter.new(result)
+      printer.print(STDOUT, options)
+    end
+
+    res
+  end
+
+  def self.memprof
+    require 'memprof'
+    Memprof.start
+    begin
+      res = yield
+    rescue Exception
+      puts "Profiling aborted"
+      raise $!
+    ensure
+      Memprof.stop
+      print Memprof.stats
+    end
+
+    res
+  end
+end

data/lib/rbbt/util/misc/exceptions.rb

@@ -0,0 +1,11 @@
+class ParameterException < Exception; end
+class FieldNotFoundError < Exception;end
+class Aborted < Exception; end
+class TryAgain < Exception; end
+class ClosedStream < Exception; end
+class KeepLocked < Exception
+  attr_accessor :payload
+  def initialize(payload)
+    @payload = payload
+  end
+end

data/lib/rbbt/util/misc/format.rb

@@ -0,0 +1,170 @@
+module Misc
+  COLOR_LIST = %w(#BC80BD #CCEBC5 #FFED6F #8DD3C7 #FFFFB3 #BEBADA #FB8072 #80B1D3 #FDB462 #B3DE69 #FCCDE5 #D9D9D9)
+
+  def self.colors_for(list)
+    unused = COLOR_LIST.dup
+
+    used = {}
+    colors = list.collect do |elem|
+      if used.include? elem
+        used[elem]
+      else
+        color = unused.shift
+        used[elem]=color
+        color
+      end
+    end
+
+    [colors, used]
+  end
+
+  def self.format_paragraph(text, size = 80, indent = 0, offset = 0)
+    i = 0
+    re = /((?:\n\s*\n\s*)|(?:\n\s*(?=\*)))/
+      text.split(re).collect do |paragraph|
+      i += 1
+      str = if i % 2 == 1
+              words = paragraph.gsub(/\s+/, "\s").split(" ")
+              lines = []
+              line = " "*offset
+              word = words.shift
+              while word
+                word = word[0..size-indent-offset-4] + '...' if word.length >= size - indent - offset
+                while word and Log.uncolor(line).length + Log.uncolor(word).length <= size - indent
+                  line << word << " "
+                  word = words.shift
+                end
+                lines << ((" " * indent) << line[0..-2])
+                line = ""
+              end
+              (lines * "\n")
+            else
+              paragraph
+            end
+      offset = 0
+      str
+      end*""
+  end
+
+  def self.format_definition_list_item(dt, dd, size = 80, indent = 20, color = :yellow)
+    dd = "" if dd.nil?
+    dt = dt.to_s + ":" unless dd.empty?
+    dt = Log.color color, dt if color
+    len = Log.uncolor(dt).length
+
+    if indent < 0
+      text = format_paragraph(dd, size, indent.abs+1, 0)
+      text = dt << "\n" << text
+    else
+      offset = len - indent
+      offset = 0 if offset < 0
+      text = format_paragraph(dd, size, indent.abs+1, offset)
+      text[0..len-1] = dt
+    end
+    text
+  end
+
+  def self.format_definition_list(defs, size = 80, indent = 20, color = :yellow)
+    entries = []
+    defs.each do |dt,dd|
+      text = format_definition_list_item(dt,dd,size,indent,color)
+      entries << text
+    end
+    entries * "\n\n"
+  end
+
+  def self.camel_case(string)
+    return string if string !~ /_/ && string =~ /[A-Z]+.*/
+      string.split(/_|(\d+)/).map{|e| 
+        (e =~ /^[A-Z]{2,}$/ ? e : e.capitalize) 
+      }.join
+  end
+
+  def self.camel_case_lower(string)
+      string.split('_').inject([]){ |buffer,e| 
+        buffer.push(buffer.empty? ? e.downcase : (e =~ /^[A-Z]{2,}$/ ? e : e.capitalize)) 
+      }.join
+  end
+
+  def self.snake_case(string)
+    return nil if string.nil?
+    string = string.to_s if Symbol === string
+    string.
+      gsub(/([A-Z]{2,})([A-Z][a-z])/,'\1_\2').
+      gsub(/([a-z])([A-Z])/,'\1_\2').
+      gsub(/\s/,'_').gsub(/[^\w_]/, '').
+      split("_").collect{|p| p.match(/[A-Z]{2,}/) ? p : p.downcase } * "_"
+  end
+
+  # source: https://gist.github.com/ekdevdes/2450285
+  # author: Ethan Kramer (https://github.com/ekdevdes)
+  def self.humanize(value, options = {})
+    if options.empty?
+      options[:format] = :sentence
+    end
+
+    values = []
+    values = value.split('_')
+    values.each_index do |index|
+      # lower case each item in array
+      # Miguel Vazquez edit: Except for acronyms
+      values[index].downcase! unless values[index].match(/[a-zA-Z][A-Z]/)
+    end
+    if options[:format] == :allcaps
+      values.each do |value|
+        value.capitalize!
+      end
+
+      if options.empty?
+        options[:seperator] = " "
+      end
+
+      return values.join " "
+    end
+
+    if options[:format] == :class
+      values.each do |value|
+        value.capitalize!
+      end
+
+      return values.join ""
+    end
+
+    if options[:format] == :sentence
+      values[0].capitalize! unless values[0].match(/[a-zA-Z][A-Z]/)
+
+      return values.join " "
+    end
+
+    if options[:format] == :nocaps
+      return values.join " "
+    end
+  end
+
+  def self.fixascii(string)
+    if string.respond_to?(:encode)
+      self.fixutf8(string).encode("ASCII-8BIT") 
+    else
+      string
+    end
+  end
+
+  def self.to_utf8(string)
+    string.encode("UTF-16BE", :invalid => :replace, :undef => :replace, :replace => "?").encode('UTF-8')
+  end
+
+  def self.fixutf8(string)
+    return nil if string.nil?
+    return string if (string.respond_to? :valid_encoding? and string.valid_encoding?) or
+    (string.respond_to? :valid_encoding and string.valid_encoding)
+
+    if string.respond_to?(:encode)
+      string.encode("UTF-16BE", :invalid => :replace, :undef => :replace, :replace => "?").encode('UTF-8')
+    else
+      require 'iconv'
+      @@ic ||= Iconv.new('UTF-8//IGNORE', 'UTF-8')
+      @@ic.iconv(string)
+    end
+  end
+
+end

data/lib/rbbt/util/misc/indiferent_hash.rb

@@ -0,0 +1,56 @@
+module IndiferentHash
+
+  def self.setup(hash)
+    hash.extend IndiferentHash 
+  end
+
+  def merge(other)
+    new = self.dup
+    IndiferentHash.setup(new)
+    other.each do |k,value|
+      new.delete k
+      new[k] = value
+    end
+    new
+  end
+
+  def [](key)
+    res = super(key) 
+    return res unless res.nil?
+
+    case key
+    when Symbol, Module
+      super(key.to_s)
+    when String
+      super(key.to_sym)
+    else
+      super(key)
+    end
+  end
+
+  def values_at(*key_list)
+    key_list.inject([]){|acc,key| acc << self[key]}
+  end
+
+  def include?(key)
+    case key
+    when Symbol, Module
+      super(key) || super(key.to_s)
+    when String
+      super(key) || super(key.to_sym)
+    else
+      super(key)
+    end
+  end
+
+  def delete(key)
+    case key
+    when Symbol, Module
+      super(key) || super(key.to_s)
+    when String
+      super(key) || super(key.to_sym)
+    else
+      super(key)
+    end
+  end
+end

data/lib/rbbt/util/misc/inspect.rb

@@ -0,0 +1,181 @@
+module Misc
+  ARRAY_MAX_LENGTH = 1000
+  STRING_MAX_LENGTH = ARRAY_MAX_LENGTH * 10
+
+  def self.sanitize_filename(filename, length = 254)
+    if filename.length > length
+      if filename =~ /(\..{2,9})$/
+        extension = $1
+      else
+        extension = ''
+      end
+
+      post_fix = "--#{filename.length}@#{length}_#{Misc.digest(filename)[0..4]}" + extension
+
+      filename = filename[0..(length - post_fix.length - 1)] << post_fix
+    else
+      filename
+    end
+    filename
+  end
+
+  def self.fingerprint(obj)
+    case obj
+    when nil
+      "nil"
+    when (defined? Step and Step)
+      obj.path || Misc.fingerprint([obj.task.name, obj.inputs])
+    when TrueClass
+      "true"
+    when FalseClass
+      "false"
+    when Symbol
+      ":" << obj.to_s
+    when String
+      if obj.length > 100
+        "'" << obj.slice(0,20) << "<...#{obj.length}...>" << obj.slice(-10,10) << " " << "'"
+      else 
+        "'" << obj << "'"
+      end
+    when (defined? AnnotatedArray and AnnotatedArray)
+      "<A: #{fingerprint Annotated.purge(obj)} #{fingerprint obj.info}>"
+    when (defined? TSV and TSV::Parser)
+      "<TSVStream:" + obj.filename + "--" << Misc.fingerprint(obj.options) << ">"
+    when IO
+      "<IO:" + (obj.respond_to?(:filename) ? obj.filename : obj.inspect) + ">"
+    when File
+      "<File:" + obj.path + ">"
+    when Array
+      if (length = obj.length) > 10
+        "[#{length}--" <<  (obj.values_at(0,1, length / 2, -2, -1).collect{|e| fingerprint(e)} * ",") << "]"
+      else
+        "[" << (obj.collect{|e| fingerprint(e) } * ",") << "]"
+      end
+    when (defined? TSV and TSV)
+      obj.with_unnamed do
+        "TSV:{"<< fingerprint(obj.all_fields|| []).inspect << ";" << fingerprint(obj.keys).inspect << "}"
+      end
+    when Hash
+      if obj.length > 10
+        "H:{"<< fingerprint(obj.keys) << ";" << fingerprint(obj.values) << "}"
+      else
+        new = "{"
+        obj.each do |k,v|
+          new << k.to_s << '=>' << fingerprint(v) << ' '
+        end
+        if new.length > 1
+           new[-1] =  "}"
+        else
+          new << '}'
+        end
+        new
+      end
+    else
+      obj.to_s
+    end
+  end
+
+
+  def self.remove_long_items(obj)
+    case
+    when IO === obj
+      remove_long_items("IO: " + obj.filename)
+    when obj.respond_to?(:path)
+      remove_long_items("File: " + obj.path)
+    when TSV::Parser === obj
+      remove_long_items("TSV Stream: " + obj.filename + " -- " << Misc.fingerprint(obj.options))
+    when TSV === obj
+      remove_long_items((obj.all_fields || []) + obj.keys.sort)
+    when (Array === obj and obj.length > ARRAY_MAX_LENGTH)
+      remove_long_items(obj[0..ARRAY_MAX_LENGTH-2] << "TRUNCATED at #{ ARRAY_MAX_LENGTH } (#{obj.length})")
+    when (Hash === obj and obj.length > ARRAY_MAX_LENGTH)
+      remove_long_items(obj.collect.compact[0..ARRAY_MAX_LENGTH-2] << ["TRUNCATED", "at #{ ARRAY_MAX_LENGTH } (#{obj.length})"])
+    when (String === obj and obj.length > STRING_MAX_LENGTH)
+      obj[0..STRING_MAX_LENGTH-1] << " TRUNCATED at #{STRING_MAX_LENGTH} (#{obj.length})"
+    when Hash === obj
+      new = {}
+      obj.each do |k,v|
+        new[k] = remove_long_items(v)
+      end
+      new
+    when Array === obj
+      obj.collect do |e| remove_long_items(e) end
+    else
+      obj
+    end
+  end
+
+  def self.filename?(string)
+    String === string and string.length > 0 and string.length < 250 and File.exists?(string)
+  end
+
+  def self.is_filename?(string)
+    return true if string.respond_to? :exists
+    return true if String === string and string.length < 265 and File.exists? string
+    return false
+  end
+
+  def self.digest(text)
+    Digest::MD5.hexdigest(text)
+  end
+
+  HASH2MD5_MAX_STRING_LENGTH = 1000
+  HASH2MD5_MAX_ARRAY_LENGTH = 100
+  def self.hash2md5(hash)
+    str = ""
+    keys = hash.keys
+    keys = keys.clean_annotations if keys.respond_to? :clean_annotations
+    keys = keys.sort_by{|k| k.to_s}
+
+    if hash.respond_to? :unnamed
+      unnamed = hash.unnamed
+      hash.unnamed = true 
+    end
+    keys.each do |k|
+      next if k == :monitor or k == "monitor" or k == :in_situ_persistence or k == "in_situ_persistence"
+      v = hash[k]
+      case
+      when TrueClass === v
+        str << k.to_s << "=>true" 
+      when FalseClass === v
+        str << k.to_s << "=>false" 
+      when Hash === v
+        str << k.to_s << "=>" << hash2md5(v)
+      when Symbol === v
+        str << k.to_s << "=>" << v.to_s
+      when (String === v and v.length > HASH2MD5_MAX_STRING_LENGTH)
+        str << k.to_s << "=>" << v[0..HASH2MD5_MAX_STRING_LENGTH] << "; #{ v.length }"
+      when String === v
+        str << k.to_s << "=>" << v
+      when (Array === v and v.length > HASH2MD5_MAX_ARRAY_LENGTH)
+        str << k.to_s << "=>[" << v[0..HASH2MD5_MAX_ARRAY_LENGTH] * "," << "; #{ v.length }]"
+      when TSV::Parser === v
+        str << remove_long_items(v)
+      when Array === v
+        str << k.to_s << "=>[" << v * "," << "]"
+      when File === v
+        str << k.to_s << "=>[File:" << v.path << "]"
+      else
+        v_ins = v.inspect
+
+        case
+        when v_ins =~ /:0x0/
+          str << k.to_s << "=>" << v_ins.sub(/:0x[a-f0-9]+@/,'')
+        else
+          str << k.to_s << "=>" << v_ins
+        end
+
+      end
+
+      str << "_" << hash2md5(v.info) if defined? Annotated and Annotated === v
+    end
+    hash.unnamed = unnamed if hash.respond_to? :unnamed
+
+    if str.empty?
+      ""
+    else
+      digest(str)
+    end
+  end
+
+end