tb 0.1 → 0.2

data/lib/tb/tsv.rb

@@ -27,6 +27,35 @@
 require 'stringio'
 
 class Tb
+  def Tb.load_tsv(filename, *header_fields, &block)
+    Tb.parse_tsv(File.read(filename), *header_fields, &block)
+  end
+
+  def Tb.parse_tsv(tsv, *header_fields)
+    aa = []
+    tsv_stream_input(tsv) {|ary|
+      aa << ary
+    }
+    aa = yield aa if block_given?
+    if header_fields.empty?
+      reader = Tb::Reader.new(aa)
+      arys = []
+      reader.each {|ary|
+        arys << ary
+      }
+      header = reader.header
+    else
+      header = header_fields
+      arys = aa
+    end
+    t = Tb.new(header)
+    arys.each {|ary|
+      ary << nil while ary.length < header.length
+      t.insert_values header, ary
+    }
+    t
+  end
+
   def Tb.tsv_stream_input(tsv)
     tsvreader = TSVReader.new(tsv)
     while ary = tsvreader.shift
@@ -53,7 +82,6 @@ class Tb
     end
 
     def close
-      @input.close
     end
   end
 

data/sample/langs.csv

@@ -0,0 +1,46 @@
+language,year
+FORTRAN,1955
+LISP,1958
+COBOL,1959
+ALGOL 58,1958
+APL,1962
+Simula,1962
+SNOBOL,1962
+BASIC,1964
+PL/I,1964
+BCPL,1967
+Logo,1968
+B,1969
+Pascal,1970
+Forth,1970
+C,1972
+Smalltalk,1972
+Prolog,1972
+ML,1973
+Scheme,1975
+SQL,1978
+C++,1980
+Objective-C,1983
+Ada,1983
+Common Lisp,1984
+Eiffel,1985
+Erlang,1986
+Perl,1987
+Tcl,1988
+Haskell,1990
+Python,1991
+Visual Basic,1991
+Ruby,1993
+Lua,1993
+CLOS,1994
+Java,1995
+Delphi,1995
+JavaScript,1995
+PHP,1995
+D,1999
+C#,2001
+F#,2002
+Groovy,2003
+Scala,2003
+Clojure,2007
+Go,2009

data/sample/tbplot

@@ -0,0 +1,293 @@
+#!/usr/bin/env ruby
+#
+# Copyright (C) 2011 Tanaka Akira  <akr@fsij.org>
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+#  1. Redistributions of source code must retain the above copyright notice, this
+#     list of conditions and the following disclaimer.
+#  2. Redistributions in binary form must reproduce the above copyright notice,
+#     this list of conditions and the following disclaimer in the documentation
+#     and/or other materials provided with the distribution.
+#  3. The name of the author may not be used to endorse or promote products
+#     derived from this software without specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+# EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+# OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+# OF SUCH DAMAGE.
+
+# plot a graph using gnuplot.
+
+# usage:
+#   tbplot [-x field] [-y field] [--shape field] [--color field] [--size field] [--facet-x field] [--facet-y field] filename 
+
+$:.unshift '/home/akr/ruby/tb/lib'
+
+require 'optparse'
+require 'tempfile'
+require 'time'
+require 'tb'
+
+def err(msg)
+  STDERR.puts msg
+  exit false
+end
+
+def gnuplot_escape_string(string)
+  string = string.dup.force_encoding("ascii-8bit") if string.respond_to? :force_encoding
+  '"' + string.gsub(/[^A-Za-z]/) {|c| sprintf("\\%03o", c.ord) } + '"'
+end
+
+class ValueChecker
+  def initialize
+    @total = 0
+    @numeric = 0
+    @numeric_min = @numeric_max = nil
+    @time = 0
+    @time_min = @time_max = nil
+    @values = {}
+  end
+  attr_reader :total, :numeric, :time
+  attr_reader :numeric_min, :numeric_max
+  attr_reader :time_min, :time_max
+  attr_reader :values
+
+  def categorical_index(val)
+    @values.fetch(val)
+  end
+
+  def check(val)
+    ret = val
+    if val.kind_of? Numeric
+      @numeric += 1
+      if @numeric == 1
+        @numeric_min = @numeric_max = val
+      else
+        @numeric_min = val if val < @numeric_min
+        @numeric_max = val if @numeric_max < val
+      end
+    elsif /\A\s*-?(\d+(\.\d*)?|\.\d+)([eE][-+]?\d+)?\s*\z/ =~ val
+      @numeric += 1
+      if @numeric == 1
+        @numeric_min = @numeric_max = val
+      else
+        @numeric_min = val if val < @numeric_min
+        @numeric_max = val if @numeric_max < val
+      end
+    elsif /\A\s*-?\d+-\d\d(-\d\d(T\d\d(:\d\d(:\d\d(\.\d+)?(Z|[+-]\d\d:?\d\d)?)?)?)?)?\s*\z/ =~ val ||
+       /\A\s*-?\d+\d\d\d\dT\d\d(\d\d(\d\d(\.\d+)?(Z|[+-]\d\d:?\d\d)?)?)?\s*\z/ =~ val
+      @time += 1
+      t = Time.parse(val).utc
+      t = tweak_timezone(t)
+      if @time == 1
+        @time_min = @time_max = t
+      else
+        @time_min = t if t < @time_min
+        @time_max = t if @time_max < t
+      end
+      ret = t.iso8601
+    end
+    @values[ret] ||= @values.size + 1
+    ret
+  end
+
+  def tweak_timezone(t)
+    # gnuplot supports UTC only.
+    # So this (not-so-valid) tweak is required.
+    unless defined? @time_offset
+      @time_offset = Time.now.utc_offset
+    end
+    t + @time_offset
+  end
+
+  def tic_format
+    if 0 < @time
+      if @time_min.year == @time_max.year &&
+         @time_min.mon  == @time_max.mon &&
+         @time_min.day  == @time_max.day
+        "%H:%M"
+      else
+        "%Y\n%m-%d"
+      end
+    else
+      "% g"
+    end
+  end
+end
+
+$x_field = nil
+$y_field = nil
+$shape_field = nil
+$color_field = nil
+$size_field = nil
+$facet_x_field = nil
+$facet_y_field = nil
+$max_size = nil
+
+op = OptionParser.new
+op.def_option('-x FIELD', 'x-field') {|f| $x_field = f }
+op.def_option('-y FIELD', 'y-field') {|f| $y_field = f }
+op.def_option('--shape=FIELD', 'shape-field') {|f| $shape_field = f }
+op.def_option('--color=FIELD', 'color-field') {|f| $color_field = f }
+op.def_option('--size=FIELD', 'size-field') {|f| $size_field = f }
+op.def_option('--facet-x=FIELD', 'facet-x-field') {|f| $facet_x_field = f }
+op.def_option('--facet-y=FIELD', 'facet-y-field') {|f| $facet_y_field = f }
+op.def_option('--shapecolor=FIELD', 'shape-field and color-field') {|f| $shape_field = $color_field = f }
+op.def_option('--max-size=MAX-SIZE', 'maximum point size') {|v| $max_size = v.to_f }
+
+op.parse!(ARGV)
+
+if !$max_size
+  if $size_field
+    $max_size = 100.0
+  else
+    $max_size = 1.0
+  end
+end
+
+tmps = {}
+
+argv = ARGV.empty? ? ['-'] : ARGV
+Tb::CatReader.open(argv) {|r|
+  header0 = r.header
+  header = header0.dup
+  [$x_field, $y_field, $shape_field, $color_field, $size_field, $facet_x_field, $facet_y_field].each {|f|
+    if f && !header.include?(f)
+      err("field not found: #{f.inspect}")
+    end
+  }
+  header -= [$x_field, $y_field, $shape_field, $color_field, $size_field, $facet_x_field, $facet_y_field].compact
+  if !$x_field
+    if header.empty?
+      err("x-field not found")
+    end
+    $x_field = header.shift
+  end
+  if !$y_field
+    if header.empty?
+      err("y-field not found")
+    end
+    $y_field = header.shift
+  end
+  checkers = {}
+  uniq_fields = [$x_field, $y_field, $shape_field, $color_field, $size_field, $facet_x_field, $facet_y_field].compact.uniq
+  r.each {|ary|
+    next if uniq_fields.any? {|f| ary[r.index_from_field(f)].nil? }
+    vs = {}
+    uniq_fields.each {|f|
+      v = ary[r.index_from_field(f)]
+      checkers[f] ||= ValueChecker.new
+      vs[f] = checkers[f].check(v)
+    }
+    x = vs[$x_field]
+    y = vs[$y_field]
+
+    shape_value = color_value = nil
+    size = 1
+    facet_x_field = facet_y_field = nil
+    if $shape_field
+      shape = vs[$shape_field]
+    end
+    if $color_field
+      color = vs[$color_field]
+    end
+    if $size_field
+      size = vs[$size_field]
+    end
+    if $facet_x_field
+      facet_x = vs[$facet_x_field]
+    end
+    if $facet_y_field
+      facet_y = vs[$facet_y_field]
+    end
+    key1 = [facet_x, facet_y]
+    key2 = [shape, color]
+    tmps[key1] ||= {}
+    tmps[key1][key2] ||= Tempfile.new('tbplot')
+    tmps[key1][key2].puts "#{x} #{y} #{size}"
+  }
+  tmps.each {|k1, h| h.each {|k2, v| v.close } }
+  gnuplot_command = ''
+  if 0 < checkers[$x_field].time || 0 < checkers[$y_field].time
+    gnuplot_command << 'set timefmt "%Y-%m-%dT%H:%M:%SZ"' << "\n"
+  end
+  if 0 < checkers[$x_field].time
+    gnuplot_command << "set xdata time\n"
+    gnuplot_command << "set format x #{gnuplot_escape_string(checkers[$x_field].tic_format)}\n"
+  end
+  if 0 < checkers[$y_field].time
+    gnuplot_command << "set ydata time\n"
+    gnuplot_command << "set format y #{gnuplot_escape_string(checkers[$y_field].tic_format)}\n"
+  end
+  gnuplot_command << "set xlabel #{gnuplot_escape_string($x_field)}\n"
+  gnuplot_command << "set ylabel #{gnuplot_escape_string($y_field)}\n"
+  if $size_field
+    pointsize = $max_size / checkers[$size_field].numeric_max
+    gnuplot_command << "set pointsize #{pointsize}\n"
+  end
+  if $facet_x_field || $facet_y_field
+    xsize = $facet_x_field ? 1.0 / checkers[$facet_x_field].values.size : 1.0
+    ysize = $facet_y_field ? 1.0 / checkers[$facet_y_field].values.size : 1.0
+    gnuplot_command << "set multiplot\n"
+    gnuplot_command << "set size #{xsize},#{ysize}\n"
+  end
+  tmps.each {|(facet_x, facet_y), h|
+    if $facet_x_field || $facet_y_field
+      xorigin = $facet_x_field ? (checkers[$facet_x_field].categorical_index(facet_x)-1.0)/checkers[$facet_x_field].values.size : 0
+      yorigin = $facet_y_field ? 1.0-(checkers[$facet_y_field].categorical_index(facet_y)-0.0)/checkers[$facet_y_field].values.size : 0
+      gnuplot_command << "set origin #{xorigin},#{yorigin}\n"
+    end
+    gnuplot_command << 'plot '
+    first = true
+    h.each {|(shape_value, color_value), tmp|
+      if $shape_field
+        shape = checkers[$shape_field].categorical_index(shape_value)
+      end
+      if $color_field
+        color = checkers[$color_field].categorical_index(color_value)
+      end
+      gnuplot_command << ",\\\n" if !first
+      gnuplot_command << gnuplot_escape_string(tmp.path)
+      gnuplot_command << ' using 1:2:3 '
+      title = []
+      if shape_value
+        title << " #{$shape_field}=#{shape_value.to_s}"
+      end
+      if color_value
+        title << " #{$color_field}=#{color_value.to_s}"
+      end
+      title = title.uniq.join(' ')
+      gnuplot_command << ' title ' << gnuplot_escape_string(title)
+      gnuplot_command << ' with points'
+      if shape
+        gnuplot_command << " pointtype " << shape.to_s # xxx: some mapping
+      else
+        gnuplot_command << " pointtype 1"
+      end
+      if color
+        gnuplot_command << " linecolor " << color.to_s # xxx: some mapping
+      else
+        gnuplot_command << " linecolor 1"
+      end
+      gnuplot_command << " pointsize variable"
+      first = false
+    }
+    gnuplot_command << "\n"
+  }
+  if $facet_x_field || $facet_y_field
+    gnuplot_command << "unset multiplot\n"
+  end
+  gnuplot_command << "pause mouse any\n"
+  tmp_gnuplot_command = Tempfile.new(['tbplot', '.gp'])
+  tmp_gnuplot_command << gnuplot_command
+  tmp_gnuplot_command.close
+  system('gnuplot', tmp_gnuplot_command.path)
+}

data/test-all-cov.rb

@@ -0,0 +1,65 @@
+require "coverage.so"
+
+def expand_tab(str, tabstop=8)
+  col = 0
+  str.gsub(/(\t+)|[^\t]+/) {
+    if $1
+      ' ' * (($1.length * tabstop) - (col + 1) % tabstop)
+    else
+      $&
+    end
+  }
+end
+
+at_exit {
+  r = Coverage.result
+  fs = r.keys.sort.reject {|f|
+    %r{lib/tb[/.]} !~ f
+  }
+  if !fs.empty?
+    if STDOUT.tty?
+      out = IO.popen(['less', '-S', '-j20', '+/ 0:'], 'w')
+    else
+      out = STDOUT
+    end
+    pat = nil
+    fs[0].chars.to_a.reverse_each {|ch|
+      if !pat
+        pat = "#{Regexp.escape(ch)}?"
+      else
+        pat = "(?:#{Regexp.escape(ch)}#{pat})?"
+      end
+    }
+    pat = Regexp.compile(pat)
+    prefix_len = fs[0].length
+    fs.each {|f|
+      l = pat.match(f).end(0)
+      prefix_len = l if l < prefix_len
+    }
+    prefix = fs[0][0, prefix_len]
+    prefix.sub!(%r{[^/]+\z}, '')
+    fs.each {|f|
+      next if %r{lib/tb[/.]} !~ f
+      f0 = f[prefix.length..-1]
+      ns = r[f]
+      max = ns.compact.max
+      w = max.to_s.length
+      fmt1 = "%s %#{w}d:%s"
+      fmt2 = "%s #{" " * w}:%s"
+      File.foreach(f).with_index {|line, i|
+        line = expand_tab(line)
+        if ns[i]
+          out.puts fmt1 % [f0, ns[i], line]
+        else
+          out.puts fmt2 % [f0, line]
+        end
+      }
+    }
+    if out != STDOUT
+      out.close
+    end
+  end
+}
+Coverage.start
+
+load 'test-all.rb'