tb 0.4 → 0.5

data/README

@@ -3,6 +3,7 @@
 tb provides a command and a library for manipulating tables:
 Unix filter like operations (grep, sort, cat, cut, ls, etc.),
 SQL like operations (join, group, etc.),
+other table operations (gsub, rename, cross, melt, unmelt, etc.),
 information extractions (git-log, svn-log, tar-tvf),
 and more.
 
@@ -78,6 +79,7 @@ Following example searches languages which name contains a non-alphabet characte
   F#,2002
 
 "grep" subcommand can take Ruby expression, instead of a regexp.
+The variable, "_", contains a hash which represents a record.
 
   % tb grep --ruby '(1990..1999).include?(_["year"].to_i)' sample/langs.csv
   language,year
@@ -145,17 +147,21 @@ There are more subcommands.
     tb join [OPTS] [TABLE1 TABLE2 ...]
     tb consecutive [OPTS] [TABLE ...]
     tb group [OPTS] KEY-FIELD1,... [TABLE ...]
-    tb cross [OPTS] HKEY-FIELD1,... VKEY-FIELD1,... [TABLE ...]
+    tb cross [OPTS] VKEY-FIELD1,... HKEY-FIELD1,... [TABLE ...]
+    tb melt KEY-FIELDS-LIST [OPTS] [TABLE ...]
+    tb nest [OPTS] NEWFIELD,OLDFIELD1,OLDFIELD2,... [TABLE ...]
+    tb unnest [OPTS] FIELD [TABLE ...]
     tb shape [OPTS] [TABLE ...]
     tb mheader [OPTS] [TABLE]
     tb crop [OPTS] [TABLE ...]
     tb ls [OPTS] [FILE ...]
+    tb tar-tvf [OPTS] [TAR-FILE ...]
     tb svn-log [OPTS] -- [SVN-LOG-ARGS]
+    tb git-log [OPTS] [GIT-DIR ...]
 
-== Command Line Tool
-
-tb command has many subcommands.
+tb help -s shows one line summary of the subcommands.
 
+  % tb help -s
   help        : Show help message of tb command.
   to-csv      : Convert a table to CSV (Comma Separated Value).
   to-tsv      : Convert a table to TSV (Tab Separated Value).
@@ -173,12 +179,17 @@ tb command has many subcommands.
   join        : Concatenate tables horizontally as left/right/full natural join.
   consecutive : Concatenate consecutive rows.
   group       : Group and aggregate rows.
-  cross       : Create a contingency table.
+  cross       : Create a cross table. (a.k.a contingency table, pivot table)
+  melt        : split value fields into records.
+  nest        : Nest fields.
+  unnest      : Unnest a field.
   shape       : Show table size.
   mheader     : Collapse multi rows header.
   crop        : Extract rectangle in a table.
   ls          : List directory entries as a table.
+  tar-tvf     : Show the file listing of tar file.
   svn-log     : Show the SVN log as a table.
+  git-log     : Show the GIT log as a table.
 
 == Install
 

data/lib/tb/cmd_git_log.rb

@@ -82,14 +82,16 @@ def (Tb::Cmd).git_log_with_git_log(dir)
       'log',
       "--pretty=#{Tb::Cmd::GIT_LOG_PRETTY_FORMAT}",
       '--decorate=full',
-        '--raw',
-        '--abbrev=40',
-        '.',
-        {:chdir=>dir}
+      '--raw',
+      '--numstat',
+      '--abbrev=40',
+      '.',
+      {:chdir=>dir}
     ]
     $stderr.puts "git command line: #{command.inspect}" if 1 <= Tb::Cmd.opt_debug
     if Tb::Cmd.opt_git_log_debug_output
-      command.last[:out] = Tb::Cmd.opt_git_log_debug_output
+      # File.realdirpath is required before Ruby 2.0.
+      command.last[:out] = File.realdirpath(Tb::Cmd.opt_git_log_debug_output)
       system(*command)
       File.open(Tb::Cmd.opt_git_log_debug_output) {|f|
         yield f
@@ -131,22 +133,34 @@ end
 
 def (Tb::Cmd).git_log_parse_commit(commit_info, files)
   commit_info = commit_info.split(/\n(?=[a-z])/)
-  Tb.csv_stream_output(files_csv="") {|gen|
-    gen << %w[mode1 mode2 hash1 hash2 status filename]
-    files.split(/\n/).each {|file_line|
-      if /\A:(\d+) (\d+) ([0-9a-f]+) ([0-9a-f]+) (\S+)\t(.+)\z/ !~ file_line
-        warn "unexpected git-log output: #{file_line.inspect}"
-        next
-      end
+  files_raw = {}
+  files_numstat = {}
+  files.split(/\n/).each {|file_line|
+    if /\A:(\d+) (\d+) ([0-9a-f]+) ([0-9a-f]+) (\S+)\t(.+)\z/ =~ file_line
       mode1, mode2, hash1, hash2, status, filename = $1, $2, $3, $4, $5, $6
       filename = git_log_unescape_filename(filename)
-      gen << [mode1, mode2, hash1, hash2, status, filename]
+      files_raw[filename] = [mode1, mode2, hash1, hash2, status]
+    elsif /\A(\d+|-)\t(\d+|-)\t(.+)\z/ =~ file_line
+      add, del, filename = $1, $2, $3
+      add = add == '-' ? nil : add.to_i
+      del = del == '-' ? nil : del.to_i
+      filename = git_log_unescape_filename(filename)
+      files_numstat[filename] = [add, del]
+    else
+      warn "unexpected git-log output (raw/numstat): #{file_line.inspect}"
+    end
+  }
+  Tb.csv_stream_output(files_csv="") {|gen|
+    gen << %w[mode1 mode2 hash1 hash2 add del status filename]
+    files_raw.each {|filename, (mode1, mode2, hash1, hash2, status)|
+      add, del = files_numstat[filename]
+      gen << [mode1, mode2, hash1, hash2, add, del, status, filename]
     }
   }
   h = {}
   commit_info.each {|s|
     if /:/ !~ s
-      warn "unexpected git-log output: #{s.inspect}"
+      warn "unexpected git-log output (header:value): #{s.inspect}"
       next
     end
     k = $`
@@ -171,7 +185,7 @@ def (Tb::Cmd).git_log_each_commit(f)
     chunk.chomp!("\x01commit-separator\x01\n")
     next if chunk.empty? # beginning of the output
     if /\nend-commit\n/ !~ chunk
-      warn "unexpected git-log output: #{chunk.inspect}"
+      warn "unexpected git-log output (end-commit): #{chunk.inspect}"
       next
     end
     commit_info, files = $`, $'

data/lib/tb/cmd_melt.rb

@@ -0,0 +1,105 @@
+# Copyright (C) 2012 Tanaka Akira  <akr@fsij.org>
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 
+#  1. Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#  2. Redistributions in binary form must reproduce the above
+#     copyright notice, this list of conditions and the following
+#     disclaimer in the documentation and/or other materials provided
+#     with the distribution.
+#  3. The name of the author may not be used to endorse or promote
+#     products derived from this software without specific prior
+#     written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+# GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Tb::Cmd.subcommands << 'melt'
+
+Tb::Cmd.default_option[:opt_melt_recnum] = nil
+Tb::Cmd.default_option[:opt_melt_regexps] = []
+Tb::Cmd.default_option[:opt_melt_list] = []
+Tb::Cmd.default_option[:opt_melt_variable_field] = 'variable'
+Tb::Cmd.default_option[:opt_melt_value_field] = 'value'
+
+def (Tb::Cmd).op_melt
+  op = OptionParser.new
+  op.banner = "Usage: tb melt KEY-FIELDS-LIST [OPTS] [TABLE ...]\n" +
+    "split value fields into records."
+  define_common_option(op, "hNo", "--no-pager")
+  op.def_option('--recnum[=FIELD]',
+                'add recnum field (default don\'t add)') {|field|
+    Tb::Cmd.opt_melt_recnum = field || 'recnum'
+  }
+  op.def_option('-R REGEXP',
+                '--melt-regexp REGEXP',
+                'regexp for melt fields') {|regexp|
+    Tb::Cmd.opt_melt_regexps << Regexp.compile(regexp)
+  }
+  op.def_option('--melt-fields FIELD,...',
+                'list of melt fields') {|fields|
+    Tb::Cmd.opt_melt_list.concat split_field_list_argument(fields)
+  }
+  op.def_option('--variable-field FIELD', 'variable field. (default: variable)') {|field|
+    Tb::Cmd.opt_melt_variable_field = field
+  }
+  op.def_option('--value-field FIELD', 'value field. (default: value)') {|field|
+    Tb::Cmd.opt_melt_value_field = field
+  }
+  op
+end
+
+def (Tb::Cmd).main_melt(argv)
+  op_melt.parse!(argv)
+  exit_if_help('melt')
+  err('no key-fields given.') if argv.empty?
+  key_fields = split_field_list_argument(argv.shift)
+  key_fields_hash = Hash[key_fields.map {|f| [f, true] }]
+  if Tb::Cmd.opt_melt_regexps.empty? && Tb::Cmd.opt_melt_list.empty?
+    melt_fields_pattern = //
+  else
+    list = Tb::Cmd.opt_melt_list + Tb::Cmd.opt_melt_regexps
+    melt_fields_pattern = /\A#{Regexp.union(list)}\z/
+  end
+  argv = ['-'] if argv.empty?
+  creader = Tb::CatReader.open(argv, Tb::Cmd.opt_N)
+  er = Tb::Enumerator.new {|y|
+    header = []
+    header << Tb::Cmd.opt_melt_recnum if Tb::Cmd.opt_melt_recnum
+    header.concat key_fields
+    header << Tb::Cmd.opt_melt_variable_field
+    header << Tb::Cmd.opt_melt_value_field
+    y.set_header header
+    creader.each_with_index {|pairs, i|
+      recnum = i + 1
+      h0 = {}
+      h0[Tb::Cmd.opt_melt_recnum] = nil if Tb::Cmd.opt_melt_recnum
+      key_fields.each {|kf|
+        h0[kf] = pairs[kf]
+      }
+      pairs.each {|f, v|
+        next if key_fields_hash[f]
+        next if melt_fields_pattern !~ f
+        h = h0.dup
+        h[Tb::Cmd.opt_melt_recnum] = recnum if Tb::Cmd.opt_melt_recnum
+        h[Tb::Cmd.opt_melt_variable_field] = f
+        h[Tb::Cmd.opt_melt_value_field] = v
+        y.yield h
+      }
+    }
+  }
+  output_tbenum(er)
+end
+

data/lib/tb/cmd_svn_log.rb

@@ -97,14 +97,23 @@ class Tb::Cmd::SVNLOGListener
         end
         @y.set_header @header
       end
+      assoc = @log.to_a.reject {|f, v| !%w[rev author date msg].include?(f) }
+      if !assoc.assoc('author')
+        assoc << ['author', '(no author)']
+      end
+      if !assoc.assoc('date')
+        assoc << ['date', '(no date)']
+      end
+      if !assoc.assoc('msg')
+        assoc << ['msg', '']
+      end
       if @log['paths']
         @log['paths'].each {|h|
-          assoc = @log.to_a.reject {|f, v| !%w[rev author date msg].include?(f) }
-          assoc += h.to_a.reject {|f, v| !%w[kind action path].include?(f) }
-          @y.yield Hash[assoc]
+          assoc2 = assoc.dup
+          assoc2.concat(h.to_a.reject {|f, v| !%w[kind action path].include?(f) })
+          @y.yield Hash[assoc2]
         }
       else
-        assoc = @log.to_a.reject {|f, v| !%w[rev author date msg].include?(f) }
         @y.yield Hash[assoc]
       end
       @log = nil

data/lib/tb/cmd_unmelt.rb

@@ -0,0 +1,106 @@
+# Copyright (C) 2012 Tanaka Akira  <akr@fsij.org>
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 
+#  1. Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#  2. Redistributions in binary form must reproduce the above
+#     copyright notice, this list of conditions and the following
+#     disclaimer in the documentation and/or other materials provided
+#     with the distribution.
+#  3. The name of the author may not be used to endorse or promote
+#     products derived from this software without specific prior
+#     written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+# GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Tb::Cmd.subcommands << 'unmelt'
+
+Tb::Cmd.default_option[:opt_unmelt_recnum] = nil
+Tb::Cmd.default_option[:opt_unmelt_keys] = []
+Tb::Cmd.default_option[:opt_unmelt_variable_field] = 'variable'
+Tb::Cmd.default_option[:opt_unmelt_value_field] = 'value'
+
+def (Tb::Cmd).op_unmelt
+  op = OptionParser.new
+  op.banner = "Usage: tb unmelt [OPTS] [TABLE ...]\n" +
+    "merge melted records into a record."
+  define_common_option(op, "hNo", "--no-pager")
+  op.def_option('--recnum[=FIELD]',
+                'use FIELD as an additional key and remove it from the result. (default: not specified)') {|field|
+    Tb::Cmd.opt_unmelt_recnum = field || 'recnum'
+  }
+  op.def_option('--keys FIELD,...', 'key fields. (default: all fields except variable and value)') {|fields|
+    Tb::Cmd.opt_unmelt_keys.concat split_field_list_argument(fields)
+  }
+  op.def_option('--variable-field FIELD', 'variable field. (default: variable)') {|field|
+    Tb::Cmd.opt_unmelt_variable_field = field
+  }
+  op.def_option('--value-field FIELD', 'value field. (default: value)') {|field|
+    Tb::Cmd.opt_unmelt_value_field = field
+  }
+  op
+end
+
+def (Tb::Cmd).main_unmelt(argv)
+  op_unmelt.parse!(argv)
+  exit_if_help('unmelt')
+  argv = ['-'] if argv.empty?
+  creader = Tb::CatReader.open(argv, Tb::Cmd.opt_N)
+  if Tb::Cmd.opt_unmelt_keys.empty?
+    key_fields = nil
+  else
+    if Tb::Cmd.opt_unmelt_recnum
+      key_fields = [Tb::Cmd.opt_unmelt_recnum]
+    else
+      key_fields = []
+    end
+    key_fields += Tb::Cmd.opt_unmelt_keys
+  end
+  er = Tb::Enumerator.new {|y|
+    creader.chunk {|pairs|
+      keys = {}
+      if key_fields
+        key_fields.each {|k|
+          keys[k] = pairs[k]
+        }
+      else
+        pairs.each_key {|k|
+          next if k == Tb::Cmd.opt_unmelt_variable_field ||
+                  k == Tb::Cmd.opt_unmelt_value_field
+          keys[k] = pairs[k]
+        }
+      end
+      keys
+    }.each {|keys, pairs_ary|
+      if Tb::Cmd.opt_unmelt_recnum
+        keys.delete Tb::Cmd.opt_unmelt_recnum
+      end
+      rec = keys.dup
+      pairs_ary.each {|pairs|
+        var = pairs[Tb::Cmd.opt_unmelt_variable_field]
+        val = pairs[Tb::Cmd.opt_unmelt_value_field]
+        if rec.has_key? var
+          y.yield rec
+          rec = keys.dup
+        end
+        rec[var] = val
+      }
+      y.yield rec
+    }
+  }
+  output_tbenum(er)
+end
+

data/lib/tb/cmdtop.rb

@@ -52,6 +52,8 @@ require 'tb/cmd_join'
 require 'tb/cmd_consecutive'
 require 'tb/cmd_group'
 require 'tb/cmd_cross'
+require 'tb/cmd_melt'
+require 'tb/cmd_unmelt'
 require 'tb/cmd_nest'
 require 'tb/cmd_unnest'
 require 'tb/cmd_shape'

data/lib/tb/enumerator.rb

@@ -55,6 +55,7 @@ class Tb::Enumerator < Enumerator
   def self.new(&enumerator_proc)
     super() {|y|
       header_proc = Thread.current[:tb_enumerator_header_proc]
+      Thread.current[:tb_enumerator_header_proc] = nil
       ty = Tb::Yielder.new(header_proc, y)
       enumerator_proc.call(ty)
       if !ty.header_proc_called

data/lib/tb/func.rb

@@ -77,24 +77,24 @@ module Tb::Func
   def Count.aggregate(count) count end
 
   module Sum; end
-  def Sum.start(value) Tb::Func.smart_numerize(value) end
+  def Sum.start(value) value.nil? ? 0 : Tb::Func.smart_numerize(value) end
   def Sum.call(v1, v2) v1 + v2 end
   def Sum.aggregate(sum) sum end
 
   module Min; end
-  def Min.start(value) [value, Tb::Func.smart_cmp_value(value)]  end
-  def Min.call(vc1, vc2) (vc1.last <=> vc2.last) <= 0 ? vc1 : vc2 end
-  def Min.aggregate(vc) vc.first end
+  def Min.start(value) value.nil? ? nil : [value, Tb::Func.smart_cmp_value(value)] end
+  def Min.call(vc1, vc2) vc1.nil? ? vc2 : vc2.nil? ? vc1 : (vc1.last <=> vc2.last) <= 0 ? vc1 : vc2 end
+  def Min.aggregate(vc) vc.nil? ? nil : vc.first end
 
   module Max; end
-  def Max.start(value) [value, Tb::Func.smart_cmp_value(value)]  end
-  def Max.call(vc1, vc2) (vc1.last <=> vc2.last) >= 0 ? vc1 : vc2 end
-  def Max.aggregate(vc) vc.first end
+  def Max.start(value) value.nil? ? nil : [value, Tb::Func.smart_cmp_value(value)] end
+  def Max.call(vc1, vc2) vc1.nil? ? vc2 : vc2.nil? ? vc1 : (vc1.last <=> vc2.last) >= 0 ? vc1 : vc2 end
+  def Max.aggregate(vc) vc.nil? ? nil : vc.first end
 
   module Avg; end
-  def Avg.start(value) [Tb::Func.smart_numerize(value), 1] end
+  def Avg.start(value) value.nil? ? [0, 0] : [Tb::Func.smart_numerize(value), 1] end
   def Avg.call(v1, v2) [v1[0] + v2[0], v1[1] + v2[1]] end
-  def Avg.aggregate(sum_count) sum_count[0] / sum_count[1].to_f end
+  def Avg.aggregate(sum_count) sum_count[1] == 0 ? nil : sum_count[0] / sum_count[1].to_f end
 
   module First; end
   def First.start(value) value end
@@ -107,12 +107,12 @@ module Tb::Func
   def Last.aggregate(value) value end
 
   module Values; end
-  def Values.start(value) [value] end
+  def Values.start(value) value.nil? ? [] : [value] end
   def Values.call(a1, a2) a1.concat a2 end
   def Values.aggregate(ary) ary.join(',') end
 
   module UniqueValues; end
-  def UniqueValues.start(value) {value => true} end
+  def UniqueValues.start(value) value.nil? ? {} : {value => true} end
   def UniqueValues.call(h1, h2) h1.update h2 end
   def UniqueValues.aggregate(hash) hash.keys.join(',') end
 

data/test/test_cmd_git_log.rb

@@ -159,4 +159,32 @@ class TestTbCmdGitLog < Test::Unit::TestCase
     assert(!log.empty?)
   end
 
+  def test_binary
+    system("git init -q")
+    File.open("foo", "w") {|f| f.print "\0\xff" }
+    system("git add foo")
+    system("git commit -q -m msg foo")
+    Tb::Cmd.main_git_log(['-o', o="o.csv"])
+    result = File.read(o)
+    tb = Tb.parse_csv(result)
+    assert_equal(1, tb.size)
+    assert_match(/,,,A,foo\n/, tb.get_record(0)["files"])
+  end
+
+  def test_subdir
+    system("git init -q")
+    File.open("foo", "w") {|f| f.print "foo" }
+    system("git add foo")
+    system("git commit -q -m msg foo")
+    Dir.mkdir("bar")
+    File.open("bar/baz", "w") {|f| f.print "baz" }
+    system("git add bar")
+    system("git commit -q -m msg bar")
+    Tb::Cmd.main_git_log(['-o', o="o.csv", "bar"])
+    result = File.read(o)
+    tb = Tb.parse_csv(result)
+    assert_equal(1, tb.size)
+    assert_not_match(/foo\n/, tb.get_record(0)["files"])
+  end
+
 end

data/test/test_cmd_group.rb

@@ -178,4 +178,121 @@ class TestTbCmdGroup < Test::Unit::TestCase
     assert(!exc.success?)
   end
 
+  def test_sum_nil
+    File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
+      k,v
+      a,2
+      b,3
+      a,
+      b,4
+    End
+    Tb::Cmd.main_group(['-o', o="o.csv", 'k', '-a', 'sum(v)', i])
+    assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
+      k,sum(v)
+      a,2
+      b,7
+    End
+  end
+
+  def test_avg_nil
+    File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
+      k,v
+      a,2
+      b,3
+      c,
+      a,
+      b,4
+      c,
+    End
+    Tb::Cmd.main_group(['-o', o="o.csv", 'k', '-a', 'avg(v)', i])
+    assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
+      k,avg(v)
+      a,2.0
+      b,3.5
+      c,
+    End
+  end
+
+  def test_min_nil
+    File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
+      k,v
+      a,9
+      b,9
+      c,
+      a,2
+      b,
+      c,
+      a,
+      b,4
+      c,
+    End
+    Tb::Cmd.main_group(['-o', o="o.csv", 'k', '-a', 'min(v)', i])
+    assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
+      k,min(v)
+      a,2
+      b,4
+      c,
+    End
+  end
+
+  def test_max_nil
+    File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
+      k,v
+      a,1
+      b,1
+      c,
+      a,2
+      b,
+      c,
+      a,
+      b,4
+      c,
+    End
+    Tb::Cmd.main_group(['-o', o="o.csv", 'k', '-a', 'max(v)', i])
+    assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
+      k,max(v)
+      a,2
+      b,4
+      c,
+    End
+  end
+
+  def test_values_nil
+    File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
+      k,v
+      a,2
+      b,
+      c,
+      a,
+      b,4
+      c,
+    End
+    Tb::Cmd.main_group(['-o', o="o.csv", 'k', '-a', 'values(v)', i])
+    assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
+      k,values(v)
+      a,2
+      b,4
+      c,""
+    End
+  end
+
+  def test_uniquevalues_nil
+    File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
+      k,v
+      a,2
+      b,
+      c,
+      a,
+      b,4
+      c,
+    End
+    Tb::Cmd.main_group(['-o', o="o.csv", 'k', '-a', 'uniquevalues(v)', i])
+    assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
+      k,uniquevalues(v)
+      a,2
+      b,4
+      c,""
+    End
+  end
+
 end

data/test/test_cmd_melt.rb

@@ -0,0 +1,172 @@
+require 'test/unit'
+require 'tb/cmdtop'
+require 'tmpdir'
+
+class TestTbCmdMelt < Test::Unit::TestCase
+  def setup
+    Tb::Cmd.reset_option
+    @curdir = Dir.pwd
+    @tmpdir = Dir.mktmpdir
+    Dir.chdir @tmpdir
+  end
+  def teardown
+    Tb::Cmd.reset_option
+    Dir.chdir @curdir
+    FileUtils.rmtree @tmpdir
+  end
+
+  def test_basic
+    File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
+      a,b,c,d
+      0,1,2,3
+      4,5,6,7
+      8,9,a,b
+      c,d,e,f
+    End
+    Tb::Cmd.main_melt(['-o', o="o.csv", 'a,b', i])
+    assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
+      a,b,variable,value
+      0,1,c,2
+      0,1,d,3
+      4,5,c,6
+      4,5,d,7
+      8,9,c,a
+      8,9,d,b
+      c,d,c,e
+      c,d,d,f
+    End
+  end
+
+  def test_recnum
+    File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
+      a,b,c,d
+      0,1,2,3
+      4,5,6,7
+      8,9,a,b
+      c,d,e,f
+    End
+    Tb::Cmd.main_melt(['-o', o="o.csv", 'a,b', '--recnum', i])
+    assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
+      recnum,a,b,variable,value
+      1,0,1,c,2
+      1,0,1,d,3
+      2,4,5,c,6
+      2,4,5,d,7
+      3,8,9,c,a
+      3,8,9,d,b
+      4,c,d,c,e
+      4,c,d,d,f
+    End
+  end
+
+  def test_recnum_value
+    File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
+      a,b,c,d
+      0,1,2,3
+      4,5,6,7
+      8,9,a,b
+      c,d,e,f
+    End
+    Tb::Cmd.main_melt(['-o', o="o.csv", 'a,b', '--recnum=rec', i])
+    assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
+      rec,a,b,variable,value
+      1,0,1,c,2
+      1,0,1,d,3
+      2,4,5,c,6
+      2,4,5,d,7
+      3,8,9,c,a
+      3,8,9,d,b
+      4,c,d,c,e
+      4,c,d,d,f
+    End
+  end
+
+  def test_variable_field
+    File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
+      a,b,c,d
+      0,1,2,3
+      4,5,6,7
+      8,9,a,b
+      c,d,e,f
+    End
+    Tb::Cmd.main_melt(['-o', o="o.csv", 'a,b', '--variable-field=foo', i])
+    assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
+      a,b,foo,value
+      0,1,c,2
+      0,1,d,3
+      4,5,c,6
+      4,5,d,7
+      8,9,c,a
+      8,9,d,b
+      c,d,c,e
+      c,d,d,f
+    End
+  end
+
+  def test_value_field
+    File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
+      a,b,c,d
+      0,1,2,3
+      4,5,6,7
+      8,9,a,b
+      c,d,e,f
+    End
+    Tb::Cmd.main_melt(['-o', o="o.csv", 'a,b', '--value-field=bar', i])
+    assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
+      a,b,variable,bar
+      0,1,c,2
+      0,1,d,3
+      4,5,c,6
+      4,5,d,7
+      8,9,c,a
+      8,9,d,b
+      c,d,c,e
+      c,d,d,f
+    End
+  end
+
+  def test_melt_regexp
+    File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
+      a,b,c,d
+      0,1,2,3
+      4,5,6,7
+      8,9,a,b
+      c,d,e,f
+    End
+    Tb::Cmd.main_melt(['-o', o="o.csv", 'a', '--melt-regexp=[bd]', i])
+    assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
+      a,variable,value
+      0,b,1
+      0,d,3
+      4,b,5
+      4,d,7
+      8,b,9
+      8,d,b
+      c,b,d
+      c,d,f
+    End
+  end
+
+  def test_melt_list
+    File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
+      a,b,c,d
+      0,1,2,3
+      4,5,6,7
+      8,9,a,b
+      c,d,e,f
+    End
+    Tb::Cmd.main_melt(['-o', o="o.csv", 'a', '--melt-fields=b,d', i])
+    assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
+      a,variable,value
+      0,b,1
+      0,d,3
+      4,b,5
+      4,d,7
+      8,b,9
+      8,d,b
+      c,b,d
+      c,d,f
+    End
+  end
+
+end

data/test/test_cmd_svn_log.rb

@@ -84,4 +84,25 @@ class TestTbCmdSvnLog < Test::Unit::TestCase
     assert_equal(1, tb.size)
     assert_match(/baz/, tb.get_record(0)["msg"])
   end
+
+  def test_no_props
+    system("svnadmin create repo")
+    File.open("repo/hooks/pre-revprop-change", "w", 0755) {|f| f.print "#!/bin/sh\nexit 0\0" }
+    system("svn co -q file://#{@tmpdir}/repo .")
+    File.open("foo", "w") {|f| f.puts "bar" }
+    system("svn add -q foo")
+    system("svn commit -q -m baz foo")
+    system("svn update -q") # update the revision of the directory.
+    system("svn propdel -q svn:author --revprop -r 1 .")
+    system("svn propdel -q svn:date --revprop -r 1 .")
+    system("svn propdel -q svn:log --revprop -r 1 .")
+    ###
+    Tb::Cmd.main_svn_log(['-o', o="o.csv"])
+    result = File.read(o)
+    tb = Tb.parse_csv(result)
+    assert_equal(1, tb.size)
+    assert_equal('(no author)', tb.get_record(0)["author"])
+    assert_equal('(no date)', tb.get_record(0)["date"])
+    assert_equal('', tb.get_record(0)["msg"])
+  end
 end

data/test/test_cmd_unmelt.rb

@@ -0,0 +1,157 @@
+require 'test/unit'
+require 'tb/cmdtop'
+require 'tmpdir'
+
+class TestTbCmdUnmelt < Test::Unit::TestCase
+  def setup
+    Tb::Cmd.reset_option
+    @curdir = Dir.pwd
+    @tmpdir = Dir.mktmpdir
+    Dir.chdir @tmpdir
+  end
+  def teardown
+    Tb::Cmd.reset_option
+    Dir.chdir @curdir
+    FileUtils.rmtree @tmpdir
+  end
+
+  def test_basic
+    File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
+      a,b,variable,value
+      0,1,x,3
+      0,1,y,7
+      4,5,x,b
+      4,5,y,f
+    End
+    Tb::Cmd.main_unmelt(['-o', o="o.csv", i])
+    assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
+      a,b,x,y
+      0,1,3,7
+      4,5,b,f
+    End
+  end
+
+  def test_json
+    File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
+      a,b,variable,value
+      0,1,x,3
+      0,1,y,7
+      4,5,x,b
+      4,5,y,f
+    End
+    Tb::Cmd.main_unmelt(['-o', o="o.json", i])
+    assert_equal(<<-"End".gsub(/\s+/, ''), File.read(o).gsub(/\s+/, ''))
+      [
+        {"a":"0", "b":"1", "x":"3", "y":"7"},
+        {"a":"4", "b":"5", "x":"b", "y":"f"}
+      ]
+    End
+  end
+
+  def test_duplicated_variable
+    File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
+      a,b,variable,value
+      0,1,x,3
+      0,1,x,4
+    End
+    Tb::Cmd.main_unmelt(['-o', o="o.csv", i])
+    assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
+      a,b,x
+      0,1,3
+      0,1,4
+    End
+  end
+
+  def test_keys
+    File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
+      a,b,variable,value
+      0,1,x,3
+      0,1,y,7
+      4,5,x,b
+      4,5,y,f
+    End
+    Tb::Cmd.main_unmelt(['-o', o="o.csv", '--keys=a', i])
+    assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
+      a,x,y
+      0,3,7
+      4,b,f
+    End
+  end
+
+  def test_recnum_noneffective
+    File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
+      recnum,a,b,variable,value
+      1,0,1,x,3
+      1,0,1,y,7
+      2,4,5,x,b
+      2,4,5,y,f
+    End
+    Tb::Cmd.main_unmelt(['-o', o="o.csv", '--keys=a,b', '--recnum', i])
+    assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
+      a,b,x,y
+      0,1,3,7
+      4,5,b,f
+    End
+  end
+
+  def test_recnum_effective
+    File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
+      recnum,a,b,variable,value
+      1,0,1,x,3
+      2,0,1,y,f
+    End
+    Tb::Cmd.main_unmelt(['-o', o="o.csv", '--keys=a,b', '--recnum', i])
+    assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
+      a,b,x,y
+      0,1,3
+      0,1,,f
+    End
+  end
+
+  def test_recnum_value
+    File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
+      r,a,b,variable,value
+      1,0,1,x,3
+      2,0,1,y,f
+    End
+    Tb::Cmd.main_unmelt(['-o', o="o.csv", '--keys=a,b', '--recnum=r', i])
+    assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
+      a,b,x,y
+      0,1,3
+      0,1,,f
+    End
+  end
+
+  def test_variable_field
+    File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
+      a,b,foo,value
+      0,1,x,3
+      0,1,y,7
+      4,5,x,b
+      4,5,y,f
+    End
+    Tb::Cmd.main_unmelt(['-o', o="o.csv", '--variable-field=foo', i])
+    assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
+      a,b,x,y
+      0,1,3,7
+      4,5,b,f
+    End
+  end
+
+  def test_value_field
+    File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
+      a,b,variable,bar
+      0,1,x,3
+      0,1,y,7
+      4,5,x,b
+      4,5,y,f
+    End
+    Tb::Cmd.main_unmelt(['-o', o="o.csv", '--value-field=bar', i])
+    assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
+      a,b,x,y
+      0,1,3,7
+      4,5,b,f
+    End
+  end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tb
 version: !ruby/object:Gem::Version
-  version: '0.4'
+  version: '0.5'
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-02-29 00:00:00.000000000 Z
+date: 2012-03-29 00:00:00.000000000 Z
 dependencies: []
 description: ! 'tb is a manipulation tool for table: CSV, TSV, JSON, etc.
 
@@ -20,6 +20,8 @@ description: ! 'tb is a manipulation tool for table: CSV, TSV, JSON, etc.
 
   SQL like operations (join, group, etc.),
 
+  other table operations (gsub, rename, cross, melt, unmelt, etc.),
+
   information extractions (git-log, svn-log, tar-tvf),
 
   and more.
@@ -48,6 +50,7 @@ files:
 - lib/tb/cmd_help.rb
 - lib/tb/cmd_join.rb
 - lib/tb/cmd_ls.rb
+- lib/tb/cmd_melt.rb
 - lib/tb/cmd_mheader.rb
 - lib/tb/cmd_nest.rb
 - lib/tb/cmd_newfield.rb
@@ -62,6 +65,7 @@ files:
 - lib/tb/cmd_to_pp.rb
 - lib/tb/cmd_to_tsv.rb
 - lib/tb/cmd_to_yaml.rb
+- lib/tb/cmd_unmelt.rb
 - lib/tb/cmd_unnest.rb
 - lib/tb/cmdmain.rb
 - lib/tb/cmdtop.rb
@@ -109,6 +113,7 @@ files:
 - test/test_cmd_help.rb
 - test/test_cmd_join.rb
 - test/test_cmd_ls.rb
+- test/test_cmd_melt.rb
 - test/test_cmd_mheader.rb
 - test/test_cmd_nest.rb
 - test/test_cmd_newfield.rb
@@ -123,6 +128,7 @@ files:
 - test/test_cmd_to_pp.rb
 - test/test_cmd_to_tsv.rb
 - test/test_cmd_to_yaml.rb
+- test/test_cmd_unmelt.rb
 - test/test_cmd_unnest.rb
 - test/test_cmdtty.rb
 - test/test_cmdutil.rb
@@ -181,6 +187,7 @@ test_files:
 - test/test_cmd_help.rb
 - test/test_cmd_join.rb
 - test/test_cmd_ls.rb
+- test/test_cmd_melt.rb
 - test/test_cmd_mheader.rb
 - test/test_cmd_nest.rb
 - test/test_cmd_newfield.rb
@@ -195,6 +202,7 @@ test_files:
 - test/test_cmd_to_pp.rb
 - test/test_cmd_to_tsv.rb
 - test/test_cmd_to_yaml.rb
+- test/test_cmd_unmelt.rb
 - test/test_cmd_unnest.rb
 - test/test_cmdtty.rb
 - test/test_cmdutil.rb