rbbt-util 5.5.68 → 5.6.0

data/lib/rbbt/resource/rake.rb

@@ -47,7 +47,8 @@ module Rake
         Rake::FileTask.clear_files
       end
     rescue
-      raise "Error in rake: #{$!.message}"
+      Log.error "Error in rake: #{$!.message}"
+      raise $!
     end
     #}
     #Process.wait(pid)

data/lib/rbbt/tsv/accessor.rb

@@ -78,8 +78,6 @@ module TSV
   end
 
   def self.extended(data)
-    #setup_chains(data)
-
     if not data.respond_to? :write
       class << data
         attr_accessor :writable
@@ -102,28 +100,79 @@ module TSV
         end
       end
     end
+  end
+
+  #{{{ TSV ENTRIES and ENTRY_KEYS
+
+  KEY_PREFIX = "__tsv_hash_"
+  ENTRIES = []
+  ENTRY_KEYS = Set.new
+  NIL_VALUE = "NIL_VALUE"
+
+  def self.entry(*entries)
+    entries = entries.collect{|entry| entry.to_s}
+    ENTRIES.concat entries
+    entries.each do |entry|
+      key = KEY_PREFIX + entry
+      ENTRY_KEYS << key
+      var_name = ("@" << entry).to_sym
+
+      TSV.send(:define_method, entry) do
+        return instance_variable_get(var_name) if instance_variables.include? var_name
+        svalue = self.send(:[], key, :entry_key)
+        value = (svalue.nil? or svalue == SERIALIZED_NIL) ? nil : TSV_SERIALIZER.load(svalue)
+        instance_variable_set(var_name, value)
+        value
+      end
+
+      TSV.send(:define_method, entry + "=") do |value|
+        instance_variable_set(var_name, value)
+        value = value.to_s if Path === value
+        self.send(:[]=, key, (value.nil? ? SERIALIZED_NIL : TSV_SERIALIZER.dump(value)), :entry_key)
+        value
+      end
 
-    if not data.respond_to? :serialized_get
-      #class << data
-      #  alias serialized_get []
-      #  alias serialized_set []=
-      #end
     end
   end
 
-  KEY_PREFIX = "__tsv_hash_"
+  entry :key_field, 
+    :type,
+    :fields,
+    :cast,
+    :identifiers,
+    :namespace,
+    :filename,
+    :serializer
 
-  ENTRIES = []
-  ENTRY_KEYS = []
+  attr_reader :serializer_module
+
+  def serializer=(serializer)
+    @serializer = serializer
+    self.send(:[]=, KEY_PREFIX + 'serializer', (serializer.nil? ? SERIALIZED_NIL : TSV_SERIALIZER.dump(serializer)), :entry_key)
+    @serializar_module = serializer.nil? ? nil : SERIALIZER_ALIAS[serializer.to_sym]
+  end
+
+
+  def serializer_module
+    @serializar_module ||= begin
+                             serializer = self.serializer
+                             serializer.nil? ? TSV::CleanSerializer : SERIALIZER_ALIAS[serializer.to_sym]
+                           end
+  end
 
-  #{{{ Chained Methods
   def empty?
     length == 0
   end
 
+  #{{{ GETTERS AND SETTERS
+
+
   def [](key, clean = false)
-    value = (self.respond_to?(:serialized_get) and not clean) ? serialized_get(key) : super(key)
-    return value if value.nil? or @unnamed or clean == :entry_key or fields.nil?
+    value = super(key)
+    return value if clean or value.nil?
+
+    value = serializer_module.load(value) if serializer_module and not TSV::CleanSerializer === serializer_module
+    return value if @unnamed or fields.nil?
 
     case type
     when :double, :list
@@ -137,12 +186,15 @@ module TSV
   end
 
   def []=(key, value, clean = false)
-    return super(key, value) if clean or not self.respond_to?(:serialized_set)
-    serialized_set(key, value)
+    if clean or serializer_module.nil? or  TSV::CleanSerializer === serializer_module or value.nil? 
+       return super(key, value)
+    else
+      return super(key, serializer_module.dump(value))
+    end
   end
 
   def keys
-    keys = super - ENTRY_KEYS
+    keys = super - ENTRY_KEYS.to_a
     return keys if @unnamed or key_field.nil?
 
     prepare_entity(keys, key_field, entity_options.merge(:dup_array => true))
@@ -168,12 +220,12 @@ module TSV
     fields = self.fields
 
     serializer = self.serializer
-    serializer_module = SERIALIZER_ALIAS[serializer] unless serializer.nil?
+    serializer_module = self.serializer_module
     super do |key, value|
       next if ENTRY_KEYS.include? key
 
       # TODO Update this to be more efficient
-      value = serializer_module.load(value) unless serializer.nil? or FalseClass === serializer
+      value = serializer_module.load(value) unless serializer_module.nil? or TSV::CleanSerializer === serializer_module
 
       # Annotated with Entity and NamedArray
       if not @unnamed
@@ -195,12 +247,12 @@ module TSV
 
   def collect
     serializer = self.serializer
-    serializer_module = SERIALIZER_ALIAS[serializer] unless serializer.nil?
+    serializer_module = self.serializer_module
     super do |key, value|
       next if ENTRY_KEYS.include? key
 
       # TODO Update this to be more efficient
-      value = serializer_module.load(value) unless serializer.nil?
+      value = serializer_module.load(value) unless serializer_module.nil? or TSV::CleanSerializer === serializer_module
 
       # Annotated with Entity and NamedArray
       if not @unnamed
@@ -338,86 +390,7 @@ module TSV
   end
 
 
-  def self.entry(*entries)
-    entries = entries.collect{|entry| entry.to_s}
-    ENTRIES.concat entries
-    entries.each do |entry|
-      key = KEY_PREFIX + entry
-      ENTRY_KEYS << key
-      line = __LINE__; self.module_eval "
-attr_accessor :#{entry}
-
-def #{ entry }
-  if not defined? @#{entry}
-    # @#{entry} = (value = self.clean_get_brackets('#{key}')).nil? ? nil : TSV_SERIALIZER.load(value)
-    @#{entry} = (value = self.send(:[], '#{key}', :entry_key)).nil? ? nil : TSV_SERIALIZER.load(value)
-  end
-  @#{entry}
-end
-
-
-if '#{entry}' == 'serializer'
-
-  def #{ entry }=(value)
-    @#{entry} = value
-    #self.tsv_clean_set_brackets '#{key}', value.nil? ? SERIALIZED_NIL : value.to_yaml
-    self.send(:[]=, '#{key}', value.nil? ? SERIALIZED_NIL : value.to_yaml, true)
-
-    return if value.nil?
-
-    self.serializer_module = SERIALIZER_ALIAS[value.to_sym]
-
-    if serializer_module.nil?
-      #class << self
-      #  alias serialized_get tsv_clean_get_brackets
-      #  alias serialized_set tsv_clean_set_brackets
-      #end
-
-    else
-      class << self
-
-        define_method :serialized_get do |key|
-          return nil unless self.include? key
-          res = self.send(:[], key, true)
-          return res if res.nil?
-          self.serializer_module.load(res)
-        end
-
-        define_method :serialized_set do |key, value|
-          if value.nil?
-            self.send(:[]=, key, value, true)
-            #tsv_clean_set_brackets key, value
-          else
-            self.send(:[]=, key, self.serializer_module.dump(value), true)
-            #tsv_clean_set_brackets key, self.serializer_module.dump(value)
-          end
-        end
-      end
-    end
-
-  end
-else
-  def #{ entry }=(value)
-    @#{entry} = value
-    self.send(:[]=, '#{key}', value.nil? ? SERIALIZED_NIL : value.to_yaml, true)
-    #self.tsv_clean_set_brackets '#{key}', value.nil? ? SERIALIZED_NIL : value.to_yaml
-  end
-end
-  ", __FILE__, line
-    end
-  end
-
-  entry :key_field, 
-    :type,
-    :fields,
-    :cast,
-    :identifiers,
-    :namespace,
-    :filename,
-    :serializer
-
   def fields
-    #@fields ||= TSV_SERIALIZER.load(self.tsv_clean_get_brackets("__tsv_hash_fields") || SERIALIZED_NIL)
     @fields ||= TSV_SERIALIZER.load(self.send(:[], "__tsv_hash_fields", :entry_key) || SERIALIZED_NIL)
     if true or @fields.nil? or @unnamed
       @fields
@@ -427,14 +400,12 @@ end
   end
 
   def namespace=(value)
-    #self.tsv_clean_set_brackets "__tsv_hash_namespace", value.nil? ? SERIALIZED_NIL : value.to_yaml
     self.send(:[]=, "__tsv_hash_namespace", value.nil? ? SERIALIZED_NIL : value.to_yaml, true)
     @namespace = value
     @entity_options = nil
   end
 
   def fields=(value)
-    #self.tsv_clean_set_brackets "__tsv_hash_fields", value.nil? ? SERIALIZED_NIL : value.to_yaml
     self.send(:[]=, "__tsv_hash_fields", value.nil? ? SERIALIZED_NIL : value.to_yaml, true)
     @fields = value
     @named_fields = nil
@@ -563,8 +534,10 @@ end
 Key field = #{key_field || "*No key field*"}
 Fields = #{fields ? Misc.fingerprint(fields) : "*No field info*"}
 Type = #{type}
+Serializer = #{serializer.inspect}
 Size = #{size}
 namespace = #{namespace}
+identifiers = #{Misc.fingerprint identifiers}
 Example:
   - #{key} -- #{Misc.fingerprint values }
       EOF

data/lib/rbbt/tsv/parser.rb

@@ -164,17 +164,14 @@ module TSV
 
     def add_to_data_no_merge_double(data, keys, values)
       keys.each do |key|
-        data[key] = values unless data.include? key
+        next if data.include? key
+        data[key] = values 
       end
     end
 
     def add_to_data_merge(data, keys, values)
       keys.uniq.each do |key|
         if data.include? key
-          #data[key] = data[key].zip(values).collect do |old, new|
-          #  old.concat new
-          #  old
-          #end
           new = data[key]
           new.each_with_index do |old, i|
             old.concat values[i]

data/lib/rbbt/tsv/serializers.rb

@@ -1,4 +1,9 @@
 module TSV
+  class CleanSerializer
+    def self.dump(o); o end
+    def self.load(o); o end
+  end
+
   class IntegerSerializer
     def self.dump(i); [i].pack("l"); end
     def self.load(str); str.unpack("l").first; end
@@ -78,6 +83,7 @@ module TSV
     :list => StringArraySerializer,
     :flat => StringArraySerializer,
     :double => StringDoubleArraySerializer,
+    :clean => CleanSerializer,
     :tsv => TSVSerializer,
     :marshal_tsv => TSVMarshalSerializer
   }

data/lib/rbbt/tsv/util.rb

@@ -85,8 +85,14 @@ module TSV
     new = {}
     case type
     when :double
-      through do |k,v|
-        new[k] = v.collect{|e| e.first}
+      if block_given?
+        through do |k,v|
+          new[k] = v.collect{|e| yield e}
+        end
+      else
+        through do |k,v|
+          new[k] = v.collect{|e| e.first}
+        end
       end
     when :flat
       through do |k,v|

data/lib/rbbt/util/R.rb

@@ -68,6 +68,12 @@ source('#{UTIL}');
     end
   end
 
+  def self.tsv(file, options = {})
+    options = Misc.add_defaults :header_hash => '', :sep => / +/, :type => :list, :key_field => 'ID'
+    key_field = Misc.process_options options, :key_field
+    clean = CMD.cmd('grep -v WARNING', :in => file, :pipe => true)
+    TSV.open(clean, options).tap{|tsv| tsv.key_field = key_field }
+  end
 end
 
 module TSV

data/lib/rbbt/util/cmd.rb

@@ -33,9 +33,12 @@ module CMD
 
         if $? and not $?.success?
           Log.debug{"Raising exception" if log}
-          exception = CMDError.new "Command [#{@pid}] #{@cmd} failed with error status #{$?.exitstatus}"
-          original_close
-          raise exception
+          exception = CMDError.new "Command [#{@pid}] '#{@cmd}' failed with error status #{$?.exitstatus}"
+          begin
+            original_close
+          ensure
+            raise exception
+          end
         end
       end
     end
@@ -97,7 +100,7 @@ module CMD
   end
 
   def self.cmd(cmd, options = {}, &block)
-    options[:stderr] ||= Log::DEBUG
+    options = Misc.add_defaults options, :stderr => Log::DEBUG
     in_content = options.delete(:in)
     stderr     = options.delete(:stderr)
     pipe       = options.delete(:pipe)

data/lib/rbbt/util/misc.rb

@@ -251,7 +251,7 @@ module Misc
       ":" << obj.to_s
     when String
       if obj.length > 100
-        "'" << obj[0..20-1] << "<...#{obj.length}...>" << obj[-10..-1] << " " << "'"
+        "'" << obj.slice(0,20) << "<...#{obj.length}...>" << obj.slice(-10,10) << " " << "'"
       else 
         "'" << obj << "'"
       end
@@ -259,7 +259,7 @@ module Misc
       "<A: #{fingerprint Annotated.purge(obj)} #{fingerprint obj.info}>"
     when Array
       if (length = obj.length) > 10
-        "[#{length} --" <<  (obj.values_at(0,1, length / 2, -2, -1).collect{|e| fingerprint(e)} * ",") << "]"
+        "[#{length}--" <<  (obj.values_at(0,1, length / 2, -2, -1).collect{|e| fingerprint(e)} * ",") << "]"
       else
         "[" << (obj.collect{|e| fingerprint(e) } * ",") << "]"
       end
@@ -709,7 +709,7 @@ end
       end
   end
 
-  def self.benchmark(repeats = 1)
+  def self.benchmark(repeats = 1, message = nil)
     require 'benchmark'
     res = nil
     begin
@@ -718,7 +718,11 @@ end
           res = yield
         end
       end
-      puts "Benchmark for #{ repeats } repeats"
+      if message
+        puts "#{message }: #{ repeats } repeats"
+      else
+        puts "Benchmark for #{ repeats } repeats"
+      end
       puts measure
     rescue Exception
       puts "Benchmark aborted"
@@ -1015,8 +1019,10 @@ end
   end
 
   def self.fixutf8(string)
+    return nil if string.nil?
     return string if (string.respond_to? :valid_encoding? and string.valid_encoding?) or
     (string.respond_to? :valid_encoding and string.valid_encoding)
+
     if string.respond_to?(:encode)
       string.encode("UTF-16BE", :invalid => :replace, :undef => :replace, :replace => "?").encode('UTF-8')
     else

data/lib/rbbt/util/open.rb

@@ -80,11 +80,11 @@ module Open
              end
 
     begin
-      CMD.cmd("wget '#{ url }'", options.merge(
-        '-O' => '-', 
-        :pipe => pipe, 
-        :stderr => stderr
-      ))
+      wget_options = options.merge( '-O' => '-')
+      wget_options[:pipe] = pipe unless pipe.nil?
+      wget_options[:stderr] = stderr unless stderr.nil?
+
+      CMD.cmd("wget '#{ url }'", wget_options)
     rescue
      STDERR.puts $!.backtrace.inspect
      raise OpenURLError, "Error reading remote url: #{ url }.\n#{$!.message}"
@@ -384,7 +384,7 @@ module Open
 
     FileUtils.mkdir_p File.dirname(file)
     case
-    when content.nil?
+    when block_given?
       begin
         File.open(file, mode) do |f| 
           yield f
@@ -393,6 +393,8 @@ module Open
         FileUtils.rm file if File.exists? file
         raise $!
       end
+    when content.nil?
+      File.open(file, mode){|f| f.write "" }
     when String === content
       file_write(file, content, mode)
     else