tb 0.1 → 0.2

data/README

@@ -1,9 +1,133 @@
 = tb - manipulation tool for table: CSV, TSV, etc.
 
-== Usage
+tb provides a command and a library for manipulating tables:
+Unix filter like operations (grep, sort, cat, cut, ls, etc.),
+SQL like operations (join, group, etc.),
+and more.
+
+== Example
+
+There is a CSV file for programming languages and their birth year in
+sample/ directory in tb package.
+
+  % head sample/langs.csv
+  language,year
+  FORTRAN,1955
+  LISP,1958
+  COBOL,1959
+  ALGOL 58,1958
+  APL,1962
+  Simula,1962
+  SNOBOL,1962
+  BASIC,1964
+  PL/I,1964
+
+"tb" command has many subcommands.
+"sort" subcommand sort a CSV file.
+You don't need to care header: header is retained as is.
+
+  % tb sort sample/langs.csv|head
+  language,year
+  ALGOL 58,1958
+  APL,1962
+  Ada,1983
+  B,1969
+  BASIC,1964
+  BCPL,1967
+  C,1972
+  C#,2001
+  C++,1980
+
+"sort" subcommand takes -f option to specify a field to sort.
+You don't need to count the position of the field.
+Also, the comparison method used in tb is smart to sort numbers correctly.
+
+  % tb sort -f year sample/langs.csv|head
+  language,year
+  FORTRAN,1955
+  LISP,1958
+  ALGOL 58,1958
+  COBOL,1959
+  APL,1962
+  SNOBOL,1962
+  Simula,1962
+  BASIC,1964
+  PL/I,1964
+
+"grep" subcommand search CSV file.
+
+  % tb grep R sample/langs.csv
+  language,year
+  FORTRAN,1955
+  Ruby,1993
+
+"grep" subcommand takes -f with field name and -v to show non-matching rows.
+You don't need to care field separators (comma) to match.
+Following example searches languages which name contains a non-alphabet character.
+
+  % tb grep -vf language '\A[A-Za-z]*\z' sample/langs.csv |cat
+  language,year
+  ALGOL 58,1958
+  PL/I,1964
+  C++,1980
+  Objective-C,1983
+  Common Lisp,1984
+  Visual Basic,1991
+  C#,2001
+  F#,2002
+
+"grep" subcommand can take Ruby expression, instead of a regexp.
+
+  % tb grep --ruby '(1990..1999).include?(_["year"].to_i)' sample/langs.csv
+  language,year
+  Haskell,1990
+  Python,1991
+  Visual Basic,1991
+  Ruby,1993
+  Lua,1993
+  CLOS,1994
+  Java,1995
+  Delphi,1995
+  JavaScript,1995
+  PHP,1995
+  D,1999
+
+"cut" subcommand extract one or more fields.
+This is similar to "cut" command of Unix and projection of relational algebra.
+
+  % tb cut language sample/langs.csv |head
+  language
+  FORTRAN
+  LISP
+  COBOL
+  ALGOL 58
+  APL
+  Simula
+  SNOBOL
+  BASIC
+  PL/I
+
+"group" subcommand groups rows for specified field.
+-a option specifies aggregation expression to aggregate the grouped rows.
+
+  % tb group year -a count -a 'values(language)' sample/langs.csv |head
+  year,count,values(language)
+  1955,1,FORTRAN
+  1958,2,"LISP,ALGOL 58"
+  1959,1,COBOL
+  1962,3,"APL,Simula,SNOBOL"
+  1964,2,"BASIC,PL/I"
+  1967,1,BCPL
+  1968,1,Logo
+  1969,1,B
+  1970,2,"Pascal,Forth"
+
+There are more subcommands.
+"help" subcommand shows list of subcommand.
 
   % tb help
   Usage:
+    tb help [OPTS] [SUBCOMMAND]
     tb csv [OPTS] [TABLE]
     tb tsv [OPTS] [TABLE]
     tb json [OPTS] [TABLE]
@@ -12,23 +136,50 @@
     tb grep [OPTS] REGEXP [TABLE]
     tb gsub [OPTS] REGEXP STRING [TABLE]
     tb sort [OPTS] [TABLE]
-    tb select [OPTS] FIELD,... [TABLE]
+    tb cut [OPTS] FIELD,... [TABLE]
     tb rename [OPTS] SRC,DST,... [TABLE]
     tb newfield [OPTS] FIELD RUBY-EXP [TABLE]
     tb cat [OPTS] [TABLE ...]
     tb join [OPTS] [TABLE ...]
-    tb group [OPTS] [TABLE]
-    tb cross [OPTS] [TABLE]
+    tb group [OPTS] KEY-FIELD1,... [TABLE]
+    tb cross [OPTS] HKEY-FIELD1,... VKEY-FIELD1,... [TABLE]
     tb shape [OPTS] [TABLE ...]
     tb mheader [OPTS] [TABLE]
     tb crop [OPTS] [TABLE]
 
-  
+== Command Line Tool
+
+tb command has many subcommands.
+
+  help     : show help message of tb command.
+  csv      : convert a table to CSV (Comma Separated Value).
+  tsv      : convert a table to TSV (Tab Separated Value).
+  json     : convert a table to JSON (JavaScript Object Notation).
+  yaml     : convert a table to YAML (YAML Ain't a Markup Language).
+  pp       : convert a table to pretty printed format.
+  grep     : search rows using regexp or ruby expression.
+  gsub     : substitute cells.
+  sort     : sort rows.
+  cut      : select columns.
+  rename   : rename field names.
+  newfield : add a field.
+  cat      : concatenate tables vertically.
+  join     : concatenate tables horizontally as left/right/full natural join.
+  group    : group and aggregate rows.
+  cross    : create a contingency table.
+  shape    : show table size.
+  mheader  : collapse multi rows header.
+  crop     : extract rectangle in a table.
 
 == Install
 
   gem install tb
 
+== Links
+
+* ((<source repository on github|URL:https://github.com/akr/tb>))
+* ((<tb on rubygems.org|URL:http://rubygems.org/gems/tb>))
+
 == Author
 
 Tanaka Akira <akr@fsij.org>

data/bin/tb

@@ -24,1114 +24,6 @@
 # IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
 # OF SUCH DAMAGE.
 
-require 'tb'
-require 'optparse'
-require 'enumerator'
+require 'tb/cmdtop'
 
-def main(argv)
-  subcommand = argv.shift
-  case subcommand
-  when 'help', '-h' then main_help(argv)
-  when 'csv' then main_csv(argv)
-  when 'tsv' then main_tsv(argv)
-  when 'json' then main_json(argv)
-  when 'yaml' then main_yaml(argv)
-  when 'pp' then main_pp(argv)
-  when 'grep' then main_grep(argv)
-  when 'gsub' then main_gsub(argv)
-  when 'sort' then main_sort(argv)
-  when 'select' then main_select(argv)
-  when 'rename' then main_rename(argv)
-  when 'newfield' then main_newfield(argv)
-  when 'cat' then main_cat(argv)
-  when 'join' then main_join(argv)
-  when 'group' then main_group(argv)
-  when 'cross' then main_cross(argv)
-  when 'shape' then main_shape(argv)
-  when 'mheader' then main_mheader(argv)
-  when 'crop' then main_crop(argv)
-  when nil
-    err "Usage: tb subcommand args..."
-  else
-    err "unexpected subcommand: #{subcommand.inspect}"
-  end
-end
-
-def usage(status)
-  print <<'End'
-Usage:
-  tb csv [OPTS] [TABLE]
-  tb tsv [OPTS] [TABLE]
-  tb json [OPTS] [TABLE]
-  tb yaml [OPTS] [TABLE]
-  tb pp [OPTS] [TABLE]
-  tb grep [OPTS] REGEXP [TABLE]
-  tb gsub [OPTS] REGEXP STRING [TABLE]
-  tb sort [OPTS] [TABLE]
-  tb select [OPTS] FIELD,... [TABLE]
-  tb rename [OPTS] SRC,DST,... [TABLE]
-  tb newfield [OPTS] FIELD RUBY-EXP [TABLE]
-  tb cat [OPTS] [TABLE ...]
-  tb join [OPTS] [TABLE ...]
-  tb group [OPTS] [TABLE]
-  tb cross [OPTS] [TABLE]
-  tb shape [OPTS] [TABLE ...]
-  tb mheader [OPTS] [TABLE]
-  tb crop [OPTS] [TABLE]
-End
-  exit status
-end
-
-def main_help(argv)
-  subcommand = argv.shift
-  case subcommand
-  when 'csv' then puts op_csv
-  when 'tsv' then puts op_tsv
-  when 'json' then puts op_json
-  when 'yaml' then puts op_yaml
-  when 'pp' then puts op_pp
-  when 'grep' then puts op_grep
-  when 'gsub' then puts op_gsub
-  when 'sort' then puts op_sort
-  when 'select' then puts op_select
-  when 'rename' then puts op_rename
-  when 'newfield' then puts op_newfield
-  when 'cat' then puts op_cat
-  when 'join' then puts op_join
-  when 'group' then puts op_group
-  when 'cross' then puts op_cross
-  when 'shape' then puts op_shape
-  when 'mheader' then puts op_mheader
-  when 'crop' then puts op_crop
-  when nil
-    usage(true)
-  else
-    err "unexpected subcommand: #{subcommand.inspect}"
-  end
-end
-
-$opt_N = nil
-$opt_debug = 0
-$opt_no_pager = nil
-
-def op_csv
-  op = OptionParser.new
-  op.banner = 'Usage: tb csv [OPTS] [TABLE]'
-  op.def_option('-h', 'show help message') { puts op; exit 0 }
-  op.def_option('-N', 'use numeric field name') { $opt_N = true }
-  op.def_option('--no-pager', 'don\'t use pager') { $opt_no_pager = true }
-  op
-end
-
-def op_tsv
-  op = OptionParser.new
-  op.banner = 'Usage: tb tsv [OPTS] [TABLE]'
-  op.def_option('-h', 'show help message') { puts op; exit 0 }
-  op.def_option('-N', 'use numeric field name') { $opt_N = true }
-  op.def_option('--no-pager', 'don\'t use pager') { $opt_no_pager = true }
-  op
-end
-
-def op_json
-  op = OptionParser.new
-  op.banner = 'Usage: tb json [OPTS] [TABLE]'
-  op.def_option('-h', 'show help message') { puts op; exit 0 }
-  op.def_option('-N', 'use numeric field name') { $opt_N = true }
-  op.def_option('--no-pager', 'don\'t use pager') { $opt_no_pager = true }
-  op
-end
-
-def op_yaml
-  op = OptionParser.new
-  op.banner = 'Usage: tb yaml [OPTS] [TABLE]'
-  op.def_option('-h', 'show help message') { puts op; exit 0 }
-  op.def_option('-N', 'use numeric field name') { $opt_N = true }
-  op.def_option('--no-pager', 'don\'t use pager') { $opt_no_pager = true }
-  op
-end
-
-def op_pp
-  op = OptionParser.new
-  op.banner = 'Usage: tb pp [OPTS] [TABLE]'
-  op.def_option('-h', 'show help message') { puts op; exit 0 }
-  op.def_option('-N', 'use numeric field name') { $opt_N = true }
-  op.def_option('--no-pager', 'don\'t use pager') { $opt_no_pager = true }
-  op
-end
-
-$opt_grep_e = nil
-$opt_grep_ruby = nil
-$opt_grep_f = nil
-$opt_grep_v = nil
-def op_grep
-  op = OptionParser.new
-  op.banner = 'Usage: tb grep [OPTS] REGEXP [TABLE]'
-  op.def_option('-h', 'show help message') { puts op; exit 0 }
-  op.def_option('-N', 'use numeric field name') { $opt_N = true }
-  op.def_option('-f FIELD', 'search field') {|field| $opt_grep_f = field }
-  op.def_option('-e REGEXP', 'predicate written in ruby.  A hash is given as _.  no usual regexp argument.') {|pattern| $opt_grep_e = pattern }
-  op.def_option('--ruby RUBY-EXP', 'specify a regexp.  no usual regexp argument.') {|ruby_exp| $opt_grep_ruby = ruby_exp }
-  op.def_option('-v', 'ouput the records which doesn\'t match') { $opt_grep_v = true }
-  op.def_option('--no-pager', 'don\'t use pager') { $opt_no_pager = true }
-  op
-end
-
-$opt_gsub_e = nil
-$opt_gsub_f = nil
-def op_gsub
-  op = OptionParser.new
-  op.banner = 'Usage: tb gsub [OPTS] REGEXP STRING [TABLE]'
-  op.def_option('-h', 'show help message') { puts op; exit 0 }
-  op.def_option('-N', 'use numeric field name') { $opt_N = true }
-  op.def_option('-f FIELD', 'search field') {|field| $opt_gsub_f = field }
-  op.def_option('-e REGEXP', 'predicate written in ruby.  A hash is given as _.  no usual regexp argument.') {|pattern| $opt_gsub_e = pattern }
-  op.def_option('--no-pager', 'don\'t use pager') { $opt_no_pager = true }
-  op
-end
-
-$opt_sort_f = nil
-def op_sort
-  op = OptionParser.new
-  op.banner = 'Usage: tb sort [OPTS] [TABLE]'
-  op.def_option('-h', 'show help message') { puts op; exit 0 }
-  op.def_option('-N', 'use numeric field name') { $opt_N = true }
-  op.def_option('-f FIELD,...', 'specify sort keys') {|fs| $opt_sort_f = fs }
-  op.def_option('--no-pager', 'don\'t use pager') { $opt_no_pager = true }
-  op
-end
-
-$opt_select_v = nil
-def op_select
-  op = OptionParser.new
-  op.banner = 'Usage: tb select [OPTS] FIELD,... [TABLE]'
-  op.def_option('-h', 'show help message') { puts op; exit 0 }
-  op.def_option('-N', 'use numeric field name') { $opt_N = true }
-  op.def_option('-v', 'invert match') { $opt_select_v = true }
-  op.def_option('--no-pager', 'don\'t use pager') { $opt_no_pager = true }
-  op
-end
-
-def op_rename
-  op = OptionParser.new
-  op.banner = 'Usage: tb rename [OPTS] SRC,DST,... [TABLE]'
-  op.def_option('-h', 'show help message') { puts op; exit 0 }
-  op.def_option('--no-pager', 'don\'t use pager') { $opt_no_pager = true }
-  op
-end
-
-def op_newfield
-  op = OptionParser.new
-  op.banner = 'Usage: tb newfield [OPTS] FIELD RUBY-EXP [TABLE]'
-  op.def_option('-h', 'show help message') { puts op; exit 0 }
-  op.def_option('--no-pager', 'don\'t use pager') { $opt_no_pager = true }
-  op
-end
-
-def op_cat
-  op = OptionParser.new
-  op.banner = 'Usage: tb cat [OPTS] [TABLE ...]'
-  op.def_option('-h', 'show help message') { puts op; exit 0 }
-  op.def_option('-N', 'use numeric field name') { $opt_N = true }
-  op.def_option('--no-pager', 'don\'t use pager') { $opt_no_pager = true }
-  op
-end
-
-$opt_join_outer = nil
-$opt_join_outer_missing = nil
-def op_join
-  op = OptionParser.new
-  op.banner = 'Usage: tb join [OPTS] [TABLE ...]'
-  op.def_option('-h', 'show help message') { puts op; exit 0 }
-  op.def_option('-d', '--debug', 'show debug message') { $opt_debug += 1 }
-  op.def_option('-N', 'use numeric field name') { $opt_N = true }
-  op.def_option('--outer', 'outer join') { $opt_join_outer = :full }
-  op.def_option('--left', 'left outer join') { $opt_join_outer = :left }
-  op.def_option('--right', 'right outer join') { $opt_join_outer = :right }
-  op.def_option('--outer-missing=DEFAULT', 'missing value for outer join') {|missing|
-    $opt_join_outer ||= :full
-    $opt_join_outer_missing = missing
-  }
-  op.def_option('--no-pager', 'don\'t use pager') { $opt_no_pager = true }
-  op
-end
-
-$opt_group_fields = []
-def op_group
-  op = OptionParser.new
-  op.banner = 'Usage: tb group [OPTS] KEY-FIELD1,... [TABLE]'
-  op.def_option('-h', 'show help message') { puts op; exit 0 }
-  op.def_option('-a AGGREGATION-SPEC[,NEW-FIELD]',
-                '--aggregate AGGREGATION-SPEC[,NEW-FIELD]') {|arg| $opt_group_fields << arg }
-  op.def_option('--no-pager', 'don\'t use pager') { $opt_no_pager = true }
-  op
-end
-
-$opt_cross_fields = []
-$opt_cross_compact = false
-def op_cross
-  op = OptionParser.new
-  op.banner = 'Usage: tb cross [OPTS] HKEY-FIELD1,... VKEY-FIELD1,... [TABLE]'
-  op.def_option('-h', 'show help message') { puts op; exit 0 }
-  op.def_option('-a AGGREGATION-SPEC[,NEW-FIELD]',
-                '--aggregate AGGREGATION-SPEC[,NEW-FIELD]') {|arg| $opt_cross_fields << arg }
-  op.def_option('-c', '--compact', 'compact format') { $opt_cross_compact = true }
-  op.def_option('--no-pager', 'don\'t use pager') { $opt_no_pager = true }
-  op
-end
-
-def op_shape
-  op = OptionParser.new
-  op.banner = 'Usage: tb shape [OPTS] [TABLE ...]'
-  op.def_option('-h', 'show help message') { puts op; exit 0 }
-  op.def_option('-N', 'use numeric field name') { $opt_N = true }
-  op.def_option('--no-pager', 'don\'t use pager') { $opt_no_pager = true }
-  op
-end
-
-$opt_mheader_count = nil
-def op_mheader
-  op = OptionParser.new
-  op.banner = 'Usage: tb mheader [OPTS] [TABLE]'
-  op.def_option('-h', 'show help message') { puts op; exit 0 }
-  op.def_option('-c N', 'number of header records') {|arg| $opt_mheader_count = arg.to_i }
-  op.def_option('--no-pager', 'don\'t use pager') { $opt_no_pager = true }
-  op
-end
-
-$opt_crop_range = nil
-def op_crop
-  op = OptionParser.new
-  op.banner = 'Usage: tb crop [OPTS] [TABLE]'
-  op.def_option('-h', 'show help message') { puts op; exit 0 }
-  op.def_option('-r RANGE', 'range.  i.e. "2,1-4,3", "B1:D3"') {|arg| $opt_crop_range = arg }
-  op.def_option('--no-pager', 'don\'t use pager') { $opt_no_pager = true }
-  op
-end
-
-def err(msg)
-  STDERR.puts msg
-  exit 1
-end
-
-def comparison_value(v)
-  case v
-  when nil
-    []
-  when Numeric
-    [0, v]
-  when String
-    case v
-    when /\A\s*-?\d+\s*\z/
-      [0, Integer(v)]
-    when /\A\s*-?(\d+(\.\d*)?|\.\d+)([eE][-+]?\d+)?\s*\z/
-      [0, Float(v)]
-    else
-      a = []
-      v.scan(/(\d+)|\D+/) {
-        if $1
-          a << 0 << $1.to_i
-        else
-          a << 1 << $&
-        end
-      }
-      a
-    end
-  else
-    raise ArgumentError, "unexpected: #{v.inspect}"
-  end
-end
-
-def conv_to_numeric(v)
-  v = v.strip
-  if /\A-?\d+\z/ =~ v
-    v = v.to_i
-  elsif /\A-?(\d+(\.\d*)?|\.\d+)([eE][-+]?\d+)?\z/ =~ v
-    v = v.to_f
-  else
-    raise "numeric value expected: #{v.inspect}"
-  end
-  v
-end
-
-class CountAggregator
-  def initialize() @result = 0 end
-  def update(v) @result += 1 end
-  def finish() @result end
-end
-
-class SumAggregator
-  def initialize() @result = 0 end 
-  def update(v) @result += conv_to_numeric(v) if !(v.nil? || v == '') end
-  def finish() @result end
-end
-
-class AvgAggregator
-  def initialize() @sum = 0; @count = 0 end 
-  def update(v) @count += 1; @sum += conv_to_numeric(v) if !(v.nil? || v == '') end
-  def finish() @sum / @count.to_f end
-end
-
-class MaxAggregator
-  def initialize() @v = nil; @cmp = nil end 
-  def update(v)
-    cmp = comparison_value(v)
-    if @cmp == nil
-      @v, @cmp = v, cmp
-    else
-      @v, @cmp = v, cmp if (@cmp <=> cmp) < 0
-    end
-  end
-  def finish() @v end
-end
-
-class MinAggregator
-  def initialize() @v = @cmp = nil end 
-  def update(v)
-    cmp = comparison_value(v)
-    if @cmp == nil
-      @v, @cmp = v, cmp
-    else
-      @v, @cmp = v, cmp if (@cmp <=> cmp) > 0
-    end
-  end
-  def finish() @v end
-end
-
-class ValuesAggregator
-  def initialize() @result = [] end 
-  def update(v) @result << v if v end
-  def finish() @result.join(",") end
-end
-
-class UniqueValuesAggregator
-  def initialize() @result = [] end 
-  def update(v) @result << v if v end
-  def finish() @result.uniq.join(",") end
-end
-
-class Selector
-  def initialize(i, aggregator) @i = i; @agg = aggregator end
-  def update(ary) @agg.update(ary[@i]) end
-  def finish() @agg.finish end
-end
-
-def make_aggregator(spec, fs)
-  case spec
-  when 'count'
-    CountAggregator.new
-  when /\Asum\((.*)\)\z/
-    field = $1
-    i = fs.index(field)
-    raise ArgumentError, "field not found: #{field.inspect}" if !i
-    Selector.new(i, SumAggregator.new)
-  when /\Aavg\((.*)\)\z/
-    field = $1
-    i = fs.index(field)
-    raise ArgumentError, "field not found: #{field.inspect}" if !i
-    Selector.new(i, AvgAggregator.new)
-  when /\Amax\((.*)\)\z/
-    field = $1
-    i = fs.index(field)
-    raise ArgumentError, "field not found: #{field.inspect}" if !i
-    Selector.new(i, MaxAggregator.new)
-  when /\Amin\((.*)\)\z/
-    field = $1
-    i = fs.index(field)
-    raise ArgumentError, "field not found: #{field.inspect}" if !i
-    Selector.new(i, MinAggregator.new)
-  when /\Avalues\((.*)\)\z/
-    field = $1
-    i = fs.index(field)
-    raise ArgumentError, "field not found: #{field.inspect}" if !i
-    Selector.new(i, ValuesAggregator.new)
-  when /\Auniquevalues\((.*)\)\z/
-    field = $1
-    i = fs.index(field)
-    raise ArgumentError, "field not found: #{field.inspect}" if !i
-    Selector.new(i, UniqueValuesAggregator.new)
-  else
-    raise ArgumentError, "unexpected aggregation spec: #{spec.inspect}"
-  end
-end
-
-def aggregate(spec, table)
-  update, finish = make_aggregator(spec, table.list_fields)
-  table.each {|rec|
-    update.call(rec.values_at(*fs))
-  }
-  finish.call
-end
-
-def main_csv(argv)
-  op_csv.parse!(argv)
-  each_table_file(argv) {|tbl|
-    with_output {|out|
-      tbl_generate_csv(tbl, out)
-    }
-  }
-end
-
-def main_tsv(argv)
-  op_tsv.parse!(argv)
-  each_table_file(argv) {|tbl|
-    with_output {|out|
-      tbl_generate_tsv(tbl, out)
-    }
-  }
-end
-
-def main_json(argv)
-  require 'json'
-  op_json.parse!(argv)
-  argv = ['-'] if argv.empty?
-  with_output {|out|
-    out.print "["
-    sep = nil
-    argv.each {|filename|
-      sep = ",\n\n" if sep
-      tablereader_open(filename) {|tblreader|
-        tblreader.each {|ary|
-          out.print sep if sep
-          header = tblreader.header
-          h = {}
-          ary.each_with_index {|e, i|
-            h[header[i]] = e if !e.nil?
-          }
-          out.print JSON.pretty_generate(h)
-          sep = ",\n"
-        }
-      }
-    }
-    out.puts "]"
-  }
-end
-
-def main_yaml(argv)
-  require 'yaml'
-  op_yaml.parse!(argv)
-  each_table_file(argv) {|tbl|
-    ary = tbl.map {|rec| rec.to_h }
-    with_output {|out|
-      YAML.dump(ary, out)
-      out.puts
-    }
-  }
-end
-
-def main_pp(argv)
-  op_pp.parse!(argv)
-  argv.unshift '-' if argv.empty?
-  with_output {|out|
-    argv.each {|filename|
-      tablereader_open(filename) {|tblreader|
-        tblreader.each {|ary|
-          h = {}
-          ary.each_with_index {|v, i|
-            next if v.nil?
-            h[tblreader.field_from_index_ex(i)] = v
-          }
-          PP.pp h, out
-        }
-      }
-    }
-  }
-end
-
-def main_grep(argv)
-  op_grep.parse!(argv)
-  if $opt_grep_ruby
-    pred = eval("lambda {|_| #{$opt_grep_ruby} }")
-  elsif $opt_grep_e
-    re = Regexp.new($opt_grep_e)
-    pred = $opt_grep_f ? lambda {|_| re =~ _[$opt_grep_f] } :
-                         lambda {|_| _.any? {|k, v| re =~ v.to_s } }
-  else
-    re = Regexp.new(argv.shift)
-    pred = $opt_grep_f ? lambda {|_| re =~ _[$opt_grep_f] } :
-                         lambda {|_| _.any? {|k, v| re =~ v.to_s } }
-  end
-  opt_v = $opt_grep_v ? true : false
-  argv.unshift '-' if argv.empty?
-  argv.each {|filename|
-    tablereader_open(filename) {|tblreader|
-      with_table_stream_output {|gen|
-        gen.output_header tblreader.header
-        tblreader.each {|ary|
-          h = {}
-          ary.each_with_index {|str, i|
-            f = tblreader.field_from_index_ex(i)
-            h[f] = str
-          }
-          found = pred.call(h)
-          found = opt_v ^ !!(found)
-          gen << ary if found
-        }
-      }
-    }
-  }
-end
-
-def main_gsub(argv)
-  op_gsub.parse!(argv)
-  if $opt_gsub_e
-    re = Regexp.new($opt_gsub_e)
-  else
-    re = Regexp.new(argv.shift)
-  end
-  repl = argv.shift
-  filename = argv.empty? ? '-' : argv.shift
-  warn "extra arguments: #{argv.join(" ")}" if !argv.empty?
-  tablereader_open(filename) {|tblreader|
-    with_table_stream_output {|gen|
-      gen.output_header tblreader.header
-      tblreader.each {|ary|
-        if $opt_gsub_f
-	  ary2 = []
-          ary.each_with_index {|str, i|
-            f = tblreader.field_from_index_ex(i)
-	    if f == $opt_gsub_f
-	      str ||= ''
-	      ary2 << str.gsub(re, repl)
-	    else
-	      ary2 << str
-	    end
-          }
-	else
-	  ary2 = ary.map {|s|
-	    s ||= ''
-	    s.gsub(re, repl)
-	  }
-	end
-	gen << ary2
-      }
-    }
-  }
-end
-
-def main_sort(argv)
-  op_sort.parse!(argv)
-  filename = argv.empty? ? '-' : argv.shift
-  warn "extra arguments: #{argv.join(" ")}" if !argv.empty?
-  if $opt_sort_f
-    fs = split_field_list_argument($opt_sort_f)
-  else
-    fs = nil
-  end
-  tbl = load_table(filename)
-  if fs
-    blk = lambda {|rec| fs.map {|f| comparison_value(rec[f]) } }
-  else
-    blk = lambda {|rec| rec.map {|k, v| comparison_value(v) } }
-  end
-  tbl2 = tbl.reorder_records_by(&blk)
-  with_output {|out|
-    tbl_generate_csv(tbl2, out)
-  }
-end
-
-def main_select(argv)
-  op_select.parse!(argv)
-  fs = split_field_list_argument(argv.shift)
-  filename = argv.shift || '-'
-  warn "extra arguments: #{argv.join(" ")}" if !argv.empty?
-  tablereader_open(filename) {|tblreader|
-    if $opt_select_v
-      h = {}
-      fs.each {|f| h[tblreader.index_from_field(f)] = true }
-      header = nil
-      if !$opt_N
-        header = []
-        tblreader.header.each_with_index {|f, i|
-          header << f if !h[i]
-        }
-      end
-      with_table_stream_output {|gen|
-        gen.output_header(header)
-        tblreader.each {|ary|
-          values = []
-          ary.each_with_index {|v, i|
-            values << v if !h[i]
-          }
-          gen << values
-        }
-      }
-    else
-      header = tblreader.header
-      is = []
-      is = fs.map {|f| tblreader.index_from_field(f) }
-      with_table_stream_output {|gen|
-        gen.output_header(is.map {|i| tblreader.field_from_index_ex(i) })
-        tblreader.each {|ary|
-          gen << ary.values_at(*is)
-        }
-      }
-    end
-  }
-end
-
-def main_rename(argv)
-  op_rename.parse!(argv)
-  fs = split_field_list_argument(argv.shift)
-  filename = argv.shift || '-'
-  warn "extra arguments: #{argv.join(" ")}" if !argv.empty?
-  h = {}
-  fs.each_slice(2) {|sf, df| h[sf] = df }
-  tablereader_open(filename) {|tblreader|
-    header = tblreader.header
-    h.each {|sf, df|
-      unless header.include? sf
-        raise "field not defined: #{sf.inspect}"
-      end
-    }
-    renamed_header = tblreader.header.map {|f| h.fetch(f, f) }
-    with_table_stream_output {|gen|
-      gen.output_header(renamed_header)
-      tblreader.each {|ary|
-        gen << ary
-      }
-    }
-  }
-end
-
-def main_newfield(argv)
-  op_rename.parse!(argv)
-  field = argv.shift
-  rubyexp = argv.shift
-  pr = eval("lambda {|_| #{rubyexp} }")
-  filename = argv.shift || '-'
-  warn "extra arguments: #{argv.join(" ")}" if !argv.empty?
-  tablereader_open(filename) {|tblreader|
-    renamed_header = [field] + tblreader.header
-    with_table_stream_output {|gen|
-      gen.output_header(renamed_header)
-      tblreader.each {|ary|
-        h = {}
-        ary.each_with_index {|str, i|
-          f = tblreader.field_from_index_ex(i)
-          h[f] = str
-        }
-        gen << [pr.call(h), *ary]
-      }
-    }
-  }
-end
-
-def main_cat(argv)
-  op_cat.parse!(argv)
-  argv = ['-'] if argv.empty?
-  if $opt_N
-    argv.each {|filename|
-      with_table_stream_output {|gen|
-        tablereader_open(filename) {|tblreader|
-          tblreader.each {|ary|
-            gen << ary
-          }
-        }
-      }
-    }
-  else
-    readers = []
-    h = {}
-    argv.each {|filename|
-      r = tablereader_open(filename)
-      readers << r
-      r.header.each {|f|
-        h[f] = h.size if !h[f]
-      }
-    }
-    with_table_stream_output {|gen|
-      gen.output_header h.keys.sort_by {|k| h[k] }
-      readers.each {|r|
-        header = r.header.dup
-        r.each {|ary|
-          while header.length < ary.length
-            f = r.field_from_index_ex(header.length)
-            header << f
-            h[f] = h.size if !h[f]
-          end
-          ary2 = []
-          ary.each_with_index {|v, i|
-            f = r.field_from_index(i)
-            j = h.fetch(f)
-            ary2[j] = v
-          }
-          gen << ary2
-        }
-      }
-    }
-  end
-end
-
-def main_join(argv)
-  op_join.parse!(argv)
-  result = Tb.new([], [])
-  retain_left = false
-  retain_right = false
-  case $opt_join_outer
-  when :full
-    retain_left = true
-    retain_right = true
-  when :left
-    retain_left = true
-  when :right
-    retain_right = true
-  when nil
-  else
-    raise "unexpected $opt_join_outer: #{$opt_join_outer.inspect}"
-  end
-  if $opt_join_outer
-    each_table_file(argv) {|tbl|
-      STDERR.puts "shared keys: #{(result.list_fields & tbl.list_fields).inspect}" if 1 <= $opt_debug
-      result = result.natjoin2_outer(tbl, $opt_join_outer_missing, retain_left, retain_right)
-    }
-  else
-    each_table_file(argv) {|tbl|
-      STDERR.puts "shared keys: #{(result.list_fields & tbl.list_fields).inspect}" if 1 <= $opt_debug
-      result = result.natjoin2(tbl)
-    }
-  end
-  with_output {|out|
-    tbl_generate_csv(result, out)
-  }
-end
-
-def main_group(argv)
-  op_group.parse!(argv)
-  kfs = split_field_list_argument(argv.shift)
-  opt_group_fields = $opt_group_fields.map {|arg|
-    aggregation_spec, new_field = split_field_list_argument(arg)
-    new_field ||= aggregation_spec
-    [new_field, lambda {|fields| make_aggregator(aggregation_spec, fields) } ]
-  }
-  filename = argv.shift || '-'
-  warn "extra arguments: #{argv.join(" ")}" if !argv.empty?
-  h = {}
-  tablereader_open(filename) {|tblreader|
-    kis = kfs.map {|f| tblreader.index_from_field(f) }
-    result_fields = kfs + opt_group_fields.map {|nf, maker| nf }
-    tblreader.each {|ary|
-      kvs = ary.values_at(*kis)
-      if !h.include?(kvs)
-        h[kvs] = opt_group_fields.map {|nf, maker| ag = maker.call(tblreader.header); ag.update(ary); ag }
-      else
-        h[kvs].each {|ag|
-          ag.update(ary)
-        }
-      end
-    }
-    result = Tb.new(result_fields)
-    h.keys.sort_by {|k| k.map {|v| comparison_value(v) } }.each {|k|
-      a = h[k]
-      result.insert_values result_fields, k + a.map {|ag| ag.finish }
-    }
-    with_output {|out|
-      tbl_generate_csv(result, out)
-    }
-  }
-end
-
-def main_cross(argv)
-  op_cross.parse!(argv)
-  hkfs = split_field_list_argument(argv.shift)
-  vkfs = split_field_list_argument(argv.shift)
-  if $opt_cross_fields.empty?
-    opt_cross_fields = [['count', 'count']]
-  else
-    opt_cross_fields = $opt_cross_fields.map {|arg|
-      agg_spec, new_field = split_field_list_argument(arg)
-      new_field ||= agg_spec
-      [agg_spec, new_field]
-    }
-  end
-  filename = argv.shift || '-'
-  warn "extra arguments: #{argv.join(" ")}" if !argv.empty?
-  tablereader_open(filename) {|tblreader|
-    vkis = vkfs.map {|f| tblreader.index_from_field(f) }
-    hkis = hkfs.map {|f| tblreader.index_from_field(f) }
-    vset = {}
-    hset = {}
-    set = {}
-    tblreader.each {|ary|
-      vkvs = ary.values_at(*vkis)
-      hkvs = ary.values_at(*hkis)
-      vset[vkvs] = true if !vset.include?(vkvs)
-      hset[hkvs] = true if !hset.include?(hkvs)
-      if !set.include?([vkvs, hkvs])
-        set[[vkvs, hkvs]] = opt_cross_fields.map {|agg_spec, nf|
-          ag = make_aggregator(agg_spec, tblreader.header)
-          ag.update(ary)
-          ag
-        }
-      else
-        set[[vkvs, hkvs]].each {|ag|
-          ag.update(ary)
-        }
-      end
-    }
-    vary = vset.keys.sort_by {|a| a.map {|v| comparison_value(v) } }
-    hary = hset.keys.sort_by {|a| a.map {|v| comparison_value(v) } }
-    with_output {|out|
-      Tb.csv_stream_output(out) {|gen|
-        hkfs.each_with_index {|hkf, i|
-          next if $opt_cross_compact && i == hkfs.length - 1
-          row = [nil] * (vkfs.length - 1) + [hkf]
-          hary.each {|hkvs| opt_cross_fields.length.times { row << hkvs[i] } }
-          gen << row
-        }
-        if $opt_cross_compact
-          r = vkfs.dup
-          hary.each {|hkvs| r.concat([hkvs[-1]] * opt_cross_fields.length) }
-          gen << r
-        else
-          r = vkfs.dup
-          hary.each {|hkvs| r.concat opt_cross_fields.map {|agg_spec, new_field| new_field } }
-          gen << r
-        end
-        vary.each {|vkvs|
-          row = vkvs.dup
-          hary.each {|hkvs|
-            ags = set[[vkvs, hkvs]]
-            if !ags
-              opt_cross_fields.length.times { row << nil }
-            else
-              ags.each {|ag| row << ag.finish }
-            end
-          }
-          gen << row
-        }
-      }
-    }
-  }
-end
-
-def main_shape(argv)
-  op_shape.parse!(argv)
-  filenames = argv.empty? ? ['-'] : argv
-  result = Tb.new(%w[header_fields min_fields max_fields records filename])
-  filenames.each {|filename|
-    tablereader_open(filename) {|tblreader|
-      num_header_fields = tblreader.header.length
-      min_num_fields = nil
-      max_num_fields = nil
-      num_records = 0
-      tblreader.each {|ary|
-        num_records += 1
-        n = ary.length
-        if min_num_fields.nil?
-          min_num_fields = max_num_fields = n
-        else
-          min_num_fields = n if n < min_num_fields
-          max_num_fields = n if max_num_fields < n
-        end
-      }
-      result.insert({'header_fields'=>num_header_fields,
-                     'min_fields'=>min_num_fields,
-                     'max_fields'=>max_num_fields,
-                     'records'=>num_records,
-                     'filename'=>filename})
-    }
-  }
-  with_output {|out|
-    # don't use tbl_generate_csv() because the header should always outputted.
-    result.generate_csv(out)
-  }
-end
-
-def main_mheader(argv)
-  op_mheader.parse!(argv)
-  filename = argv.shift || '-'
-  warn "extra arguments: #{argv.join(" ")}" if !argv.empty?
-  header = []
-  if $opt_mheader_count
-    c = $opt_mheader_count
-    header_end_p = lambda {
-      c -= 1
-      c == 0 ? header.map {|a| a.compact.join(' ').strip } : nil
-    }
-  else
-    header_end_p = lambda {
-      h2 = header.map {|a| a.compact.join(' ').strip }.uniq
-      header.length == h2.length ? h2 : nil
-    }
-  end
-  with_table_stream_output {|gen|
-    Tb::Reader.open(filename, {:numeric=>true}) {|tblreader|
-      tblreader.each {|ary|
-        if header
-          ary.each_with_index {|v,i|
-            header[i] ||= []
-            header[i] << v if header[i].empty? || header[i].last != v
-          }
-          h2 = header_end_p.call
-          if h2
-            gen << h2
-            header = nil
-          end
-        else
-          gen << ary
-        end
-      }
-    }
-  }
-  if header
-    warn "no header found."
-  end
-end
-
-def main_crop(argv)
-  op_crop.parse!(argv)
-  filename = argv.shift || '-'
-  warn "extra arguments: #{argv.join(" ")}" if !argv.empty?
-  stream = false
-  if $opt_crop_range
-    case $opt_crop_range
-    when /\A(\d+),(\d+)-(\d+),(\d+)\z/ # 1-based
-      stream = true
-      range_col1 = $1.to_i
-      range_row1 = $2.to_i
-      range_col2 = $3.to_i
-      range_row2 = $4.to_i
-    when /\A([A-Z]+)(\d+):([A-Z]+)(\d+)\z/ # 1-based
-      stream = true
-      range_col1 = decode_a1_addressing_col($1)
-      range_row1 = $2.to_i
-      range_col2 = decode_a1_addressing_col($3)
-      range_row2 = $4.to_i
-    else
-      raise ArgumentError, "unexpected range argument: #{$opt_crop_range.inspect}"
-    end
-  end
-  if stream
-    with_table_stream_output {|gen|
-      Tb::Reader.open(filename, {:numeric=>true}) {|tblreader|
-        rownum = 1
-	tblreader.each {|ary|
-	  if range_row2 < rownum
-	    break
-	  end
-	  if range_row1 <= rownum
-	    if range_col2 < ary.length
-	      ary[range_col2..-1] = []
-	    end
-	    if 1 < range_col1
-	      ary[0...(range_col1-1)] = []
-	    end
-	    gen << ary
-	  end
-	  rownum += 1
-	}
-      }
-    }
-  else
-    arys = []
-    Tb::Reader.open(filename, {:numeric=>true}) {|tblreader|
-      tblreader.each {|a|
-        a.pop while !a.empty? && (a.last.nil? || a.last == '')
-	arys << a
-      }
-    }
-    arys.pop while !arys.empty? && arys.last.all? {|v| v.nil? || v == '' }
-    arys.shift while !arys.empty? && arys.first.all? {|v| v.nil? || v == '' }
-    if !arys.empty?
-      while arys.all? {|a| a.empty? || (a.first.nil? || a.first == '') }
-	arys.each {|a| a.shift }
-      end
-    end
-    with_table_stream_output {|gen|
-      arys.each {|a| gen << a }
-    }
-  end
-end
-
-def decode_a1_addressing_col(str)
-  (26**str.length-1)/25+str.tr("A-Z", "0-9A-P").to_i(26)
-end
-
-def split_field_list_argument(arg)
-  split_csv_argument(arg).map {|f| f || '' }
-end
-
-def split_csv_argument(arg)
-  Tb.csv_stream_input(arg) {|ary| return ary }
-  return []
-end
-
-def each_table_file(argv)
-  if argv.empty?
-    yield load_table('-')
-  else
-    argv.each {|filename|
-      tbl = load_table(filename)
-      yield tbl
-    }
-  end
-end
-
-def load_table(filename)
-  tablereader_open(filename) {|tblreader|
-    arys = []
-    tblreader.each {|ary|
-      arys << ary
-    }
-    header = tblreader.header
-    tbl = Tb.new(header)
-    arys.each {|ary|
-      ary << nil while ary.length < header.length
-      tbl.insert_values header, ary
-    }
-    tbl
-  }
-end
-
-def tablereader_open(filename, &b)
-  Tb::Reader.open(filename, {:numeric=>$opt_N}, &b)
-end
-
-def with_table_stream_output
-  with_output {|out|
-    Tb.csv_stream_output(out) {|gen|
-      def gen.output_header(header)
-        self << header if !$opt_N
-      end
-      yield gen
-    }
-  }
-end
-
-def tbl_generate_csv(tbl, out)
-  if $opt_N
-    header = tbl.list_fields
-    Tb.csv_stream_output(out) {|gen|
-      tbl.each {|rec|
-        gen << rec.values_at(*header)
-      }
-    }
-  else
-    tbl.generate_csv(out)
-  end
-end
-
-def tbl_generate_tsv(tbl, out)
-  if $opt_N
-    header = tbl.list_fields
-    Tb.tsv_stream_output(out) {|gen|
-      tbl.each {|rec|
-        gen << rec.values_at(*header)
-      }
-    }
-  else
-    tbl.generate_tsv(out)
-  end
-end
-
-def with_output
-  if STDOUT.tty? && !$opt_no_pager
-    IO.popen(ENV['PAGER'] || 'more', 'w') {|pager|
-      yield pager
-    }
-  else
-    yield STDOUT
-  end
-end
-
-main ARGV
+Tb::Cmd.main(ARGV)