rbbt-util 4.4.0 → 5.0.0

data/lib/rbbt/tsv/accessor.rb

@@ -1,8 +1,11 @@
 require 'rbbt/util/chain_methods'
+require 'yaml'
 module TSV
   extend ChainMethods
   self.chain_prefix = :tsv
 
+  NIL_YAML = "--- \n"
+
   attr_accessor :unnamed, :serializer_module, :entity_options
 
   def entity_options
@@ -101,7 +104,7 @@ module TSV
   end
 
   def tsv_values
-    values = values_at(*keys)
+    values = chunked_values_at(keys)
     return values if @unnamed or fields.nil?
 
     case type
@@ -188,6 +191,16 @@ module TSV
     end
   end
 
+  def chunked_values_at(keys, max = 5000)
+    Misc.ordered_divide(keys, max).inject([]) do |acc,c|
+      new = self.values_at(*c)
+      new.annotate acc if new.respond_to? :annotate and acc.empty?
+      acc.concat(new)
+    end
+  end
+
+
+
   #{{{ Sorting
 
   def tsv_sort_by(field = nil, just_keys = false, &block)
@@ -278,7 +291,7 @@ attr_accessor :#{entry}
 
 def #{ entry }
   if not defined? @#{entry}
-    @#{entry} = YAML.load(self.tsv_clean_get_brackets('#{key}') || nil.to_yaml)
+    @#{entry} = (value = self.tsv_clean_get_brackets('#{key}')).nil? ? nil : YAML.load(value)
   end
   @#{entry}
 end
@@ -288,7 +301,7 @@ if '#{entry}' == 'serializer'
 
   def #{ entry }=(value)
     @#{entry} = value
-    self.tsv_clean_set_brackets '#{key}', value.to_yaml
+    self.tsv_clean_set_brackets '#{key}', value.nil? ? NIL_YAML : value.to_yaml
 
     return if value.nil?
 
@@ -324,7 +337,7 @@ if '#{entry}' == 'serializer'
 else
   def #{ entry }=(value)
     @#{entry} = value
-    self.tsv_clean_set_brackets '#{key}', value.to_yaml
+    self.tsv_clean_set_brackets '#{key}', value.nil? ? NIL_YAML : value.to_yaml
   end
 end
 "
@@ -341,7 +354,7 @@ end
     :serializer
 
   def fields
-    @fields ||= YAML.load(self.tsv_clean_get_brackets("__tsv_hash_fields") || nil.to_yaml)
+    @fields ||= YAML.load(self.tsv_clean_get_brackets("__tsv_hash_fields") || "--- \n")
     if @fields.nil? or @unnamed
       @fields
     else

data/lib/rbbt/tsv/attach/util.rb

@@ -9,10 +9,20 @@ module TSV
     other.with_unnamed do
       with_unnamed do
         through do |key, values|
+          self[key] = [] if self[key].nil?
           if other.include? key
-            new_values = other[key].values_at *field_positions
+            case
+            when other.type == :flat
+              new_values = [other[key]]
+            when other.type == :single
+              new_values = [other[key]]
+            else
+              new_values = other[key].values_at *field_positions
+            end
+
             new_values.collect!{|v| [v]}     if     type == :double and not other.type == :double
             new_values.collect!{|v| v.nil? ? nil : (other.type == :single ? v : v.first)} if not type == :double and     other.type == :double
+
             self[key] = self[key].concat new_values
           else
             if type == :double
@@ -248,7 +258,7 @@ module TSV
               if values.nil?
                 nil
               else
-                next_index.values_at(*values).flatten.collect
+                next_index.values_at(*values).flatten.collect.to_a
               end
             end
             current_index.fields = [next_key]

data/lib/rbbt/tsv/manipulate.rb

@@ -181,7 +181,14 @@ module TSV
         if not traverser.new_field_names.nil? 
           case type
           when :double, :list
-            NamedArray.setup value, traverser.new_field_names, key, entity_options
+            if value.frozen?
+              Log.warn "Value frozen: #{ value }"
+            end
+            if value.nil?
+              nil
+            else
+              NamedArray.setup value, traverser.new_field_names, key, entity_options
+            end
           when :flat, :single
             Misc.prepare_entity(value, traverser.new_field_names.first, entity_options)
           end
@@ -280,7 +287,7 @@ module TSV
     elems.sort_by{|k,v| v}.collect{|k,v| k}
   end
 
-  def select(method = nil)
+  def select(method = nil, &block)
     new = TSV.setup({}, :key_field => key_field, :fields => fields, :type => type, :filename => filename, :identifiers => identifiers)
 
     new.key_field = key_field
@@ -320,24 +327,48 @@ module TSV
       end
     when String === method
       if block_given?
-        with_unnamed do
-          case
-          when (method == key_field or method == :key)
-            through do |key, values|
-              new[key] = values if yield(key)
+        case 
+        when block.arity == 1
+          with_unnamed do
+            case
+            when (method == key_field or method == :key)
+              through do |key, values|
+                new[key] = values if yield(key)
+              end
+            when (type == :single or type == :flat)
+              through do |key, value|
+                new[key] = value if yield(value)
+              end
+            else
+              pos = identify_field method
+              through do |key, values|
+                new[key] = values if yield(values[pos])
+              end
             end
-          when (type == :single or type == :flat)
-            through do |key, value|
-              new[key] = value if yield(value)
-            end
-          else
-            pos = identify_field method
-            through do |key, values|
-              new[key] = values if yield(values[pos])
+          end
+        when block.arity == 2
+          with_unnamed do
+            case
+            when (method == key_field or method == :key)
+              through do |key, values|
+                new[key] = values if yield(key, key)
+              end
+            when (type == :single or type == :flat)
+              through do |key, value|
+                new[key] = value if yield(key, value)
+              end
+            else
+              pos = identify_field method
+              through do |key, values|
+                new[key] = values if yield(key, values[pos])
+              end
             end
+
           end
 
         end
+
+
       else
         with_unnamed do
           through do |key, values|
@@ -405,6 +436,22 @@ module TSV
     new
   end
 
+  def column(field, cast = nil)
+    new = slice(field)
+
+    new.with_unnamed do
+      new.each do |k,v|
+        nv = v.first 
+        nv = nv.send(cast) unless cast.nil?
+        new[k] = nv
+      end
+    end
+
+    new.type = :single
+
+    new
+  end
+
   def process(field, &block)
     field_pos = identify_field field
 
@@ -438,10 +485,16 @@ module TSV
       when type == :flat
         self[key] = new_values
       else
-        values[field_pos].replace new_values
+        if (String === values[field_pos] and String === new_values) or
+          (Array === values[field_pos] and Array === new_values) 
+           values[field_pos].replace new_values
+        else
+          values[field_pos] = new_values
+        end
         self[key] = values
       end
     end
+    self
   end
 
   def add_field(name = nil)

data/lib/rbbt/tsv/parser.rb

@@ -65,11 +65,12 @@ module TSV
     def get_values_single_from_flat(parts)
       return parts.shift, parts.first if field_positions.nil? and key_position.nil?
       if key_position == 0
-        [parts.shift, parts]
+        [parts.shift, parts.first]
       else
         key = parts.shift
-        [parts, [key]]
+        [parts, key]
       end
+
     end
 
     def get_values_single(parts)

data/lib/rbbt/tsv/util.rb

@@ -35,13 +35,14 @@ module TSV
   def self.get_filename(file)
     case
     when String === file
-      filename = file
+      file
+    when file.respond_to?(:filename)
+      file.filename
     when file.respond_to?(:gets)
-      filename = file.filename if file.respond_to? :filename
+      nil
     else
-      raise "Cannot get stream from: #{file.inspect}"
+      raise "Cannot get filename from: #{file.inspect}"
     end
-    filename
   end
 
   def self.get_stream(file, open_options = {})

data/lib/rbbt/util/chain_methods.rb

@@ -32,8 +32,8 @@ module ChainMethods
 
             class << base; self; end.module_eval do
               methods.each do |new_method|
-                original = new_method.to_s.sub(prefix.to_s + '_', '')
-                clean_method = prefix.to_s + '_clean_' + original
+                original = new_method.to_s.sub(prefix.to_s << '_', '')
+                clean_method = prefix.to_s << '_clean_' << original
 
                 original = "[]" if original == "get_brackets"
                 original = "[]=" if original == "set_brackets"
@@ -66,12 +66,12 @@ module ChainMethods
           end
         end
       end
-  end
+    end
 
-  base.chain_prefix = base.to_s.downcase.to_sym
-end
+    base.chain_prefix = base.to_s.downcase.to_sym
+  end
 
-def self.extended(base)
-  chain_methods_extended(base)
-end
+  def self.extended(base)
+    chain_methods_extended(base)
+  end
 end

data/lib/rbbt/util/cmd.rb

@@ -159,7 +159,7 @@ module CMD
         exit(-1)
       rescue Exception
         Log.debug("CMDError: #{$!.message}") if log
-        ddd $!.backtrace if log
+        Log.debug("Backtrace: \n" + $!.backtrace * "\n") if log
         raise CMDError, $!.message
       end
     }

data/lib/rbbt/util/misc.rb

@@ -9,7 +9,12 @@ require 'digest/md5'
 module Misc
   class FieldNotFoundError < StandardError;end
 
-  COLOR_LIST = %w(red green blue black yellow pink purple)
+  def self.humanize(string)
+    string.gsub(/([a-z])([A-Z])/,'\1_\2').downcase
+  end
+
+  COLOR_LIST = %w(#BC80BD #CCEBC5 #FFED6F #8DD3C7 #FFFFB3 #BEBADA #FB8072 #80B1D3 #FDB462 #B3DE69 #FCCDE5 #D9D9D9)
+
   def self.colors_for(list)
     unused = COLOR_LIST.dup
 
@@ -98,8 +103,21 @@ module Misc
   def self.prepare_entity(entity, field, options = {})
     return entity unless String === entity or Array === entity
     options ||= {}
+
     dup_array = options.delete :dup_array
-    entity = Entity.formats[field].setup(((entity.frozen? and not entity.nil?) ? entity.dup : ((Array === entity and dup_array) ? entity.collect{|e| e.nil? ? e : e.dup} : entity) ), options.merge({:format => field})) if defined?(Entity) and Entity.respond_to?(:formats) and Entity.formats.include? field
+
+    if defined?(Entity) and Entity.respond_to?(:formats) and Entity.formats.include? field
+      params = options.dup
+
+      params[:format] ||= params.delete "format"
+      params.merge!(:format => field) unless params.include?(:format) and not ((f = params[:format]).nil? or (String === f and f.empty?))
+
+      entity = Entity.formats[field].setup(
+        ((entity.frozen? and not entity.nil?) ? entity.dup : ((Array === entity and dup_array) ? entity.collect{|e| e.nil? ? e : e.dup} : entity) ),
+        params
+      ) 
+    end
+
     entity
   end
 
@@ -128,7 +146,7 @@ module Misc
     when (Array === obj and obj.length > ARRAY_MAX_LENGTH)
       remove_long_items(obj[0..ARRAY_MAX_LENGTH-2] << "TRUNCATED at #{ ARRAY_MAX_LENGTH } (#{obj.length})")
     when (Hash === obj and obj.length > ARRAY_MAX_LENGTH)
-      remove_long_items(obj.compact[0..ARRAY_MAX_LENGTH-2] << ["TRUNCATED", "at #{ ARRAY_MAX_LENGTH } (#{obj.length})"])
+      remove_long_items(obj.collect.compact[0..ARRAY_MAX_LENGTH-2] << ["TRUNCATED", "at #{ ARRAY_MAX_LENGTH } (#{obj.length})"])
     when (String === obj and obj.length > STRING_MAX_LENGTH)
       obj[0..STRING_MAX_LENGTH-1] << " TRUNCATED at #{STRING_MAX_LENGTH} (#{obj.length})"
     when Hash === obj
@@ -218,9 +236,10 @@ module Misc
     if Hash === values.last
       extra = values.pop
       inputs = Misc.zip2hash(keys, values)
-      inputs.delete_if{|k,v| v.nil?}
+      inputs.delete_if{|k,v| v.nil? or (String === v and v.empty?)}
       inputs = Misc.add_defaults inputs, extra
-      inputs.delete_if{|k,v| not keys.include? k}
+      inputs.delete_if{|k,v| not keys.include?(k) and not (Symbol === k ? keys.include?(k.to_s) : keys.include?(k.to_sym))}
+      inputs
     else
       Misc.zip2hash(keys, values)
     end
@@ -461,16 +480,21 @@ end
     new
   end
 
-  def self.array2hash(array)
+  def self.array2hash(array, default = nil)
     hash = {}
     array.each do |key, value|
+      value = default.dup if value.nil? and not default.nil?
       hash[key] = value
     end
     hash
   end
 
   def self.zip2hash(list1, list2)
-    array2hash(list1.zip(list2))
+    hash = {}
+    list1.each_with_index do |e,i|
+      hash[e] = list2[i]
+    end
+    hash
   end
 
   def self.process_to_hash(list)
@@ -506,7 +530,46 @@ end
     res
   end
 
-  def self.profile
+  def self.profile_html(options = {})
+    require 'ruby-prof'
+    RubyProf.start
+    begin
+      res = yield
+    rescue Exception
+      puts "Profiling aborted"
+      raise $!
+    ensure
+      result = RubyProf.stop
+      printer = RubyProf::MultiPrinter.new(result)
+      TmpFile.with_file do |dir|
+        FileUtils.mkdir_p dir unless File.exists? dir
+        printer.print(:path => dir, :profile => 'profile')
+        CMD.cmd("firefox  -no-remote  '#{ dir }'")
+      end
+    end
+
+    res
+  end
+
+  def self.profile_graph(options = {})
+    require 'ruby-prof'
+    RubyProf.start
+    begin
+      res = yield
+    rescue Exception
+      puts "Profiling aborted"
+      raise $!
+    ensure
+      result = RubyProf.stop
+      #result.eliminate_methods!([/annotated_array_clean_/])
+      printer = RubyProf::GraphPrinter.new(result)
+      printer.print(STDOUT, options)
+    end
+
+    res
+  end
+
+  def self.profile(options = {})
     require 'ruby-prof'
     RubyProf.start
     begin
@@ -516,8 +579,9 @@ end
       raise $!
     ensure
       result = RubyProf.stop
+      #result.eliminate_methods!([/annotated_array_clean_/])
       printer = RubyProf::FlatPrinter.new(result)
-      printer.print(STDOUT, 0)
+      printer.print(STDOUT, options)
     end
 
     res
@@ -544,8 +608,10 @@ end
     begin
       yield
     rescue
+      Log.warn("Insisting after exception: #{$!.message}")
       try += 1
       retry if try < times
+      raise $!
     end
   end
 
@@ -561,6 +627,30 @@ end
     }.compact * "#"
   end
 
+  def self.GET_params2hash(string)
+    hash = {}
+    string.split('&').collect{|item|
+      key, value = item.split("=").values_at 0, 1
+      hash[key] = value.nil? ? "" : CGI.unescape(value)
+    }
+    hash
+  end
+
+  def self.hash2GET_params(hash)
+    hash.sort_by{|k,v| k.to_s}.collect{|k,v| 
+      next unless %w(Symbol String Float Fixnum Integer TrueClass FalseClass Module Class Object Array).include? v.class.to_s
+      v = case 
+          when Symbol === v
+            ":" << v.to_s
+          when Array === v
+            v * ","
+          else
+            CGI.escape(v.to_s)
+          end
+      [ Symbol === k ? k.to_s : k,  v] * "="
+    }.compact * "&"
+  end
+
   def self.path_relative_to(basedir, path)
     path = File.expand_path(path)
     basedir = File.expand_path(basedir)
@@ -662,6 +752,7 @@ end
 
       new_options[key] = value 
     end
+
     new_options
   end
 
@@ -694,7 +785,7 @@ end
       when String === v
         str << k.to_s << "=>" << v
       when (Array === v and v.length > HASH2MD5_MAX_ARRAY_LENGTH)
-        str << k.to_s << "=>[" << v[0..HASH2MD5_MAX_ARRAY_LENGTH] * "," << "]"
+        str << k.to_s << "=>[" << v[0..HASH2MD5_MAX_ARRAY_LENGTH] * "," << "; #{ v.length }]"
       when Array === v
         str << k.to_s << "=>[" << v * "," << "]"
       else
@@ -719,9 +810,9 @@ end
 
   def self.process_options(hash, *keys)
     if keys.length == 1
-      hash.delete keys.first.to_sym
+      hash.include?(keys.first.to_sym) ? hash.delete(keys.first.to_sym) : hash.delete(keys.first.to_s) 
     else
-      keys.collect do |key| hash.delete(key.to_sym) || hash.delete(key.to_s) end
+      keys.collect do |key| hash.include?(key.to_sym) ? hash.delete(key.to_sym) : hash.delete(key.to_s) end
     end
   end
 
@@ -815,7 +906,7 @@ end
   # element in each chunk iteratively.
   def self.divide(array, num)
     chunks = []
-    num.times do chunks << [] end
+    num.to_i.times do chunks << [] end
     array.each_with_index{|e, i|
       c = i % num
       chunks[c] << e
@@ -823,6 +914,20 @@ end
     chunks
   end
 
+  # Divides the array into +num+ chunks of the same size by placing one
+  # element in each chunk iteratively.
+  def self.ordered_divide(array, num)
+    last = array.length - 1
+    chunks = []
+    current = 0
+    while current <= last
+      next_current = [last, current + num - 1].min
+      chunks << array[current..next_current]
+      current = next_current + 1
+    end
+    chunks
+  end
+
   def self.zip_fields(array)
     return [] if array.empty?
     array[0].zip(*array[1..-1])