rbbt-util 5.2.4 → 5.3.0

data/lib/rbbt/tsv/attach.rb

@@ -145,7 +145,7 @@ module TSV
 
     unless TSV === other
       other_identifier_files = other.identifier_files if other.respond_to? :identifier_files
-      other = TSV.open(other, :persist => options[:persist_input] == true) unless TSV === other 
+      other = TSV.open(other, :persist => options[:persist_input] == true)
       other.identifiers = other_identifier_files 
     end
 
@@ -161,6 +161,7 @@ module TSV
 
     case
     when key_field == other.key_field
+      Log.debug "Attachment with same key: #{other.key_field}"
       attach_same_key other, fields
     when (not in_namespace and self.fields.include?(other.key_field))
       Log.debug "Found other's key field: #{other.key_field}"
@@ -171,6 +172,7 @@ module TSV
     else
       index = TSV.find_traversal(self, other, options)
       raise "Cannot traverse identifiers" if index.nil?
+      Log.debug "Attachment with index: #{other.key_field}"
       attach_index other, index, fields
     end
     Log.debug("Attachment of fields:#{fields.inspect} from #{other.filename.inspect} finished.")

data/lib/rbbt/tsv/attach/util.rb

@@ -10,10 +10,16 @@ module TSV
       with_unnamed do
         through do |key, values|
           self[key] = [] if self[key].nil?
+          current = self[key]
+          current = [current] unless Array === current
           if other.include? key
             case
             when other.type == :flat
-              new_values = [other[key]]
+              if type == :flat
+                new_values = other[key]
+              else
+                new_values = [other[key]]
+              end
             when other.type == :single
               new_values = [other[key]]
             else
@@ -23,19 +29,23 @@ module TSV
             new_values.collect!{|v| [v]}     if     type == :double and not other.type == :double
             new_values.collect!{|v| v.nil? ? nil : (other.type == :single ? v : v.first)} if not type == :double and     other.type == :double
 
-            self[key] = self[key].concat new_values
+            new_values.flatten if type == :flat
+
+            self[key] = current.concat new_values
           else
             if type == :double
-              self[key] = self[key].concat [[]] * fields.length
+              self[key] = current.concat [[]] * fields.length
             else
-              self[key] = self[key].concat [""] * fields.length
+              self[key] = current.concat [""] * fields.length
             end
           end
         end
       end
     end
 
-    self.fields = self.fields.concat other.fields.values_at *fields
+    self.type = :list if self.type == :single
+
+    self.fields = self.fields.concat fields
   end
 
   def attach_source_key(other, source, options = {})
@@ -141,11 +151,16 @@ module TSV
                       source_key
                     end
                   else
-                    other[source_key][pos]
+                    if other.type == :flat
+                      other[source_key]
+                    else
+                      other[source_key][pos]
+                    end
                   end
                 end
                 new_values.collect!{|v| v.nil? ? [[]] : [v]}    if     type == :double and not other.type == :double
                 new_values.collect!{|v| v.nil? ? nil : (other.type == :single ? v : v.first)} if not type == :double and     other.type == :double
+                new_values.flatten! if type == :flat
                 all_new_values << new_values
               end
             end
@@ -160,6 +175,8 @@ module TSV
 
             current = self[key] || [[]] * fields.length
 
+            current = [current] unless Array === current
+
             if current.length > length
               all_new_values << current.slice!(length..current.length - 1)
             end
@@ -172,12 +189,14 @@ module TSV
 
             current += all_new_values
 
-            self[key] = current
+            self[key].replace current
           end
         end
       end
     end
 
+    self.type = :list if self.type == :single
+
     self.fields = self.fields.concat field_names
   end
 
@@ -219,7 +238,7 @@ module TSV
   end
 
   def self.build_traverse_index(files, options = {})
-    options       = Misc.add_defaults options, :in_namespace => false, :persist_input => false
+    options       = Misc.add_defaults options, :in_namespace => false, :persist_input => true
     in_namespace  = options[:in_namespace]
     persist_input = options[:persist_input]
 
@@ -237,7 +256,7 @@ module TSV
                    nil
                  else
                    Log.debug "Data index required"
-                   data_file.index :target => data_key, :fields => [data_file.key_field], :persist => false
+                   data_file.index :target => data_key, :fields => [data_file.key_field], :persist => false, :type => (data_file.type == :single ? :single : :flat)
                  end
 
     current_index = data_index
@@ -257,7 +276,12 @@ module TSV
               if values.nil?
                 nil
               else
-                next_index.values_at(*values).flatten.collect.to_a
+                new_values = next_index.values_at(*values).flatten
+                if current_index.type == :single
+                  new_values.first
+                else
+                  new_values
+                end
               end
             end
             current_index.fields = [next_key]

data/lib/rbbt/tsv/index.rb

@@ -85,8 +85,8 @@ module TSV
             values.unshift key
 
             values.uniq.each do |value|
-              case
-              when index_type == :double
+              case index_type
+              when :double
                 if not new.include? value
                   new[value] = [[key]]
                 else
@@ -94,6 +94,15 @@ module TSV
                   current[0] << key
                   new[value] = current
                 end
+              when :flat
+                if not new.include? value
+                  new[value] = [key]
+                else
+                  current = new[value]
+                  current << key
+                  new[value] = current
+                end
+
               else
                 new[value] = key unless new.include? value
               end
@@ -101,7 +110,7 @@ module TSV
           end
         end
 
-        TSV.setup(new, :serializer => index_type, :type => index_type, :filename => filename, :fields => [new_key_field], :key_field => new_fields * ", ")
+        TSV.setup(new, :type => index_type, :filename => filename, :fields => [new_key_field], :key_field => new_fields * ", ")
       end
     end
   end

data/lib/rbbt/tsv/manipulate.rb

@@ -345,6 +345,8 @@ module TSV
               end
             else
               pos = identify_field method
+              raise "Field #{ method } not identified. Available: #{ fields * ", " }" if pos.nil?
+
               through do |key, values|
                 new[key] = values if yield(values[pos])
               end
@@ -451,11 +453,32 @@ module TSV
       end
     end
 
-    new.type = :single
+    case type
+    when :double, :flat
+      new.type = :flat
+    else
+      new.type = :single
+    end
 
     new
   end
 
+  def process_key(&block)
+    new = annotate({})
+    through do |key, values|
+      key = case 
+            when block.arity == 1
+              yield(key)
+            when block.arity == 2
+              yield(key, values)
+            else
+              raise "Unexpected arity in block, must be 1, 2 or 3: #{block.arity}"
+            end
+      new[key] = values
+    end
+    new
+  end
+
   def process(field, &block)
     field_pos = identify_field field
 
@@ -498,6 +521,7 @@ module TSV
         self[key] = values
       end
     end
+
     self
   end
 

data/lib/rbbt/tsv/parser.rb

@@ -370,6 +370,7 @@ module TSV
       data.key_field = @key_field
       data.fields = @fields
       data.namespace = @namespace
+      data.cast = @cast if Symbol === @cast
       data
     end
   end

data/lib/rbbt/util/R.rb

@@ -7,7 +7,12 @@ module R
   UTIL    = File.join(LIB_DIR, 'util.R')
 
   def self.run(command, options = {})
-    cmd = "source('#{UTIL}');\n"
+    cmd =<<-EOF
+# Loading basic rbbt environment
+source('#{UTIL}');
+
+    EOF
+
     case
     when IO === command
       cmd << command.read
@@ -19,7 +24,15 @@ module R
 
     Log.debug "R Script:\n#{ cmd }"
 
-    CMD.cmd('R --vanilla --slave --quiet', options.merge(:in => cmd))
+    if options.delete :monitor
+      io = CMD.cmd('R --vanilla --slave --quiet', options.merge(:in => cmd, :pipe => true))
+      while line = io.gets
+        puts line
+      end
+      nil
+    else
+      CMD.cmd('R --vanilla --slave --quiet', options.merge(:in => cmd))
+    end
   end
 
   def self.interactive(init_file, options = {})
@@ -29,9 +42,9 @@ module R
   def self.interactive(script, options = {})
     TmpFile.with_file do |init_file|
         Open.write(init_file) do |file|
-          profile = File.join(ENV["HOME"], ".Rprofile")
-          file.puts "source('#{profile}');\n" if File.exists? profile
+          file.puts "# Loading basic rbbt environment"
           file.puts "source('#{R::UTIL}');\n"
+          file.puts 
           file.puts script
         end
         CMD.cmd("env R_PROFILE='#{init_file}' xterm R")
@@ -40,12 +53,21 @@ module R
 
   def self.ruby2R(object)
     case object
+    when nil
+      "NULL"
+    when TSV
+      #"as.matrix(data.frame(c(#{object.transpose("Field").collect{|k,v| "#{k}=" << R.ruby2R(v)}.flatten * ", "}), row.names=#{R.ruby2R object.keys}))"
+      "matrix(#{R.ruby2R object.values},dimnames=list(#{R.ruby2R object.keys}, #{R.ruby2R object.fields}))"
+    when Symbol
+      "#{ object }"
     when String
       "'#{ object }'"
-    when Fixnum
+    when Fixnum, Float
       object
     when Array
       "c(#{object.collect{|e| ruby2R(e) } * ", "})"
+    else
+      raise "Type of object not known: #{ object.inspect }"
     end
   end
 
@@ -58,13 +80,21 @@ module TSV
       Open.write(f, self.to_s)
       Log.debug(R.run(
       <<-EOF
+## Loading tsv into data
 data = rbbt.tsv('#{f}');
+
 #{script.strip}
+
+## Resaving data
 if (! is.null(data)){ rbbt.tsv.write('#{f}', data); }
       EOF
       ).read)
       open_options = Misc.add_defaults open_options, :type => :list
-      TSV.open(f, open_options) unless open_options[:ignore_output]
+      if open_options[:raw]
+        Open.read(f)
+      else
+        TSV.open(f, open_options) unless open_options[:ignore_output]
+      end
     end
   end
 

data/lib/rbbt/util/cmd.rb

@@ -105,6 +105,7 @@ module CMD
     pipe       = options.delete(:pipe)
     post       = options.delete(:post)
     log        = options.delete(:log)
+    dont_close_in        = options.delete(:dont_close_in)
 
     log = true if log.nil?
 
@@ -186,7 +187,7 @@ module CMD
           end
 
           sin.close unless sin.closed?
-          in_content.close unless in_content.closed?
+          in_content.close unless in_content.closed? or dont_close_in
         rescue
           Process.kill "INT", pid
           raise $!

data/lib/rbbt/util/color.rb

@@ -0,0 +1,250 @@
+# Copyright (c) 2007 McClain Looney
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+
+# Implements a color (r,g,b + a) with conversion to/from web format (eg #aabbcc), and
+# with a number of utilities to lighten, darken and blend values.
+class Color
+
+  attr_reader :r, :g, :b, :a
+
+  # Table for conversion to hex
+  HEXVAL = (('0'..'9').to_a).concat(('A'..'F').to_a).freeze
+  # Default value for #darken, #lighten etc.
+  BRIGHTNESS_DEFAULT = 0.2
+
+  # Constructor.  Inits to white (#FFFFFF) by default, or accepts any params
+  # supported by #parse.
+  def initialize(*args)
+    @r = 255
+    @g = 255
+    @b = 255
+    @a = 255
+
+    if args.size.between?(3,4)
+      self.r = args[0]
+      self.g = args[1]
+      self.b = args[2]
+      self.a = args[3] if args[3]
+    else
+      set(*args)
+    end
+  end
+
+  # All-purpose setter - pass in another Color, '#000000', rgb vals... whatever
+  def set(*args)
+    val = Color.parse(*args)
+    unless val.nil?
+      self.r = val.r
+      self.g = val.g
+      self.b = val.b
+      self.a = val.a
+    end
+    self
+  end
+
+  # Test for equality, accepts string vals as well, eg Color.new('aaa') == '#AAAAAA' => true
+  def ==(val)
+    val = Color.parse(val)
+    return false if val.nil?
+    return r == val.r && g == val.g && b == val.b && a == val.a
+  end
+
+  # Setters for individual channels - take 0-255 or '00'-'FF' values
+  def r=(val); @r = from_hex(val); end
+  def g=(val); @g = from_hex(val); end
+  def b=(val); @b = from_hex(val); end
+  def a=(val); @a = from_hex(val); end
+
+  # Attempt to read in a string and parse it into values
+  def self.parse(*args)
+    case args.size
+
+    when 0 then
+      return nil
+
+    when 1 then
+      val = args[0]
+
+      # Trivial parse... :-)
+      return val if val.is_a?(Color)
+
+      # Single value, assume grayscale
+      return Color.new(val, val, val) if val.is_a?(Fixnum)
+
+      # Assume string
+      str = val.to_s.upcase
+      str = str[/[0-9A-F]{3,8}/] || ''
+      case str.size
+      when 3, 4 then
+        r, g, b, a = str.scan(/[0-9A-F]/)
+      when 6,8 then
+        r, g, b, a = str.scan(/[0-9A-F]{2}/)
+      else
+        return nil
+      end
+
+      return Color.new(r,g,b,a || 255)
+
+    when 3,4 then
+      return Color.new(*args)
+
+    end
+    nil
+  end
+
+  def inspect
+    to_s(true)
+  end
+
+  def to_s(add_hash = true)
+    trans? ? to_rgba(add_hash) : to_rgb(add_hash)
+  end
+
+  def to_rgb(add_hash = true)
+    (add_hash ? '#' : '') + to_hex(r) + to_hex(g) + to_hex(b)
+  end
+
+  def to_rgba(add_hash = true)
+    to_rgb(add_hash) + to_hex(a)
+  end
+
+  def opaque?
+    @a == 255
+  end
+
+  def trans?
+    @a != 255
+  end
+
+  def grayscale?
+    @r == @g && @g == @b
+  end
+
+  # Lighten color towards white.  0.0 is a no-op, 1.0 will return #FFFFFF
+  def lighten(amt = BRIGHTNESS_DEFAULT)
+    return self if amt <= 0
+    return WHITE if amt >= 1.0
+    val = Color.new(self)
+    val.r += ((255-val.r) * amt).to_i
+    val.g += ((255-val.g) * amt).to_i
+    val.b += ((255-val.b) * amt).to_i
+    val
+  end
+
+  # In place version of #lighten
+  def lighten!(amt = BRIGHTNESS_DEFAULT)
+    set(lighten(amt))
+    self
+  end
+
+  # Darken a color towards full black.  0.0 is a no-op, 1.0 will return #000000
+  def darken(amt = BRIGHTNESS_DEFAULT)
+    return self if amt <= 0
+    return BLACK if amt >= 1.0
+    val = Color.new(self)
+    val.r -= (val.r * amt).to_i
+    val.g -= (val.g * amt).to_i
+    val.b -= (val.b * amt).to_i
+    val
+  end
+
+  # In place version of #darken
+  def darken!(amt = BRIGHTNESS_DEFAULT)
+    set(darken(amt))
+    self
+  end
+
+  # Convert to grayscale, using perception-based weighting
+  def grayscale
+    val = Color.new(self)
+    val.r = val.g = val.b = (0.2126 * val.r + 0.7152 * val.g + 0.0722 * val.b)
+    val
+  end
+
+  # In place version of #grayscale
+  def grayscale!
+    set(grayscale)
+    self
+  end
+
+  # Blend to a color amt % towards another color value, eg
+  # red.blend(blue, 0.5) will be purple, white.blend(black, 0.5) will be gray, etc.
+  def blend(other, amt)
+    other = Color.parse(other)
+    return Color.new(self) if amt <= 0 || other.nil?
+    return Color.new(other) if amt >= 1.0
+    val = Color.new(self)
+    val.r += ((other.r - val.r)*amt).to_i
+    val.g += ((other.g - val.g)*amt).to_i
+    val.b += ((other.b - val.b)*amt).to_i
+    val
+  end
+
+  # In place version of #blend
+  def blend!(other, amt)
+    set(blend(other, amt))
+    self
+  end
+
+  # Class-level version for explicit blends of two values, useful with constants
+  def self.blend(col1, col2, amt)
+    col1 = Color.parse(col1)
+    col2 = Color.parse(col2)
+    col1.blend(col2, amt)
+  end
+
+  protected
+
+  # Convert int to string hex, eg 255 => 'FF'
+  def to_hex(val)
+    HEXVAL[val / 16] + HEXVAL[val % 16]
+  end
+
+  # Convert int or string to int, eg 80 => 80, 'FF' => 255, '7' => 119
+  def from_hex(val)
+    if val.is_a?(String)
+      # Double up if single char form
+      val = val + val if val.size == 1
+      # Convert to integer
+      val = val.hex
+    end
+    # Clamp
+    val = 0 if val < 0
+    val = 255 if val > 255
+    val
+  end
+
+  public
+
+  # Some constants for general use
+  WHITE = Color.new(255,255,255).freeze
+  BLACK = Color.new(0,0,0).freeze
+
+end
+
+# "Global" method for creating Color objects, eg:
+#   new_color = rgb(params[:new_color])
+#   style="border: 1px solid <%= rgb(10,50,80).lighten %>"
+def rgb(*args)
+  Color.parse(*args)
+end
+
+