tb 0.1

data/README

@@ -0,0 +1,60 @@
+= tb - manipulation tool for table: CSV, TSV, etc.
+
+== Usage
+
+  % tb help
+  Usage:
+    tb csv [OPTS] [TABLE]
+    tb tsv [OPTS] [TABLE]
+    tb json [OPTS] [TABLE]
+    tb yaml [OPTS] [TABLE]
+    tb pp [OPTS] [TABLE]
+    tb grep [OPTS] REGEXP [TABLE]
+    tb gsub [OPTS] REGEXP STRING [TABLE]
+    tb sort [OPTS] [TABLE]
+    tb select [OPTS] FIELD,... [TABLE]
+    tb rename [OPTS] SRC,DST,... [TABLE]
+    tb newfield [OPTS] FIELD RUBY-EXP [TABLE]
+    tb cat [OPTS] [TABLE ...]
+    tb join [OPTS] [TABLE ...]
+    tb group [OPTS] [TABLE]
+    tb cross [OPTS] [TABLE]
+    tb shape [OPTS] [TABLE ...]
+    tb mheader [OPTS] [TABLE]
+    tb crop [OPTS] [TABLE]
+
+  
+
+== Install
+
+  gem install tb
+
+== Author
+
+Tanaka Akira <akr@fsij.org>
+
+== License
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+(1) Redistributions of source code must retain the above copyright notice, this
+    list of conditions and the following disclaimer.
+(2) Redistributions in binary form must reproduce the above copyright notice,
+    this list of conditions and the following disclaimer in the documentation
+    and/or other materials provided with the distribution.
+(3) The name of the author may not be used to endorse or promote products
+    derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+OF SUCH DAMAGE.
+
+(The modified BSD licence)

data/bin/tb

@@ -0,0 +1,1137 @@
+#!/usr/bin/env ruby
+#
+# Copyright (C) 2011 Tanaka Akira  <akr@fsij.org>
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+#  1. Redistributions of source code must retain the above copyright notice, this
+#     list of conditions and the following disclaimer.
+#  2. Redistributions in binary form must reproduce the above copyright notice,
+#     this list of conditions and the following disclaimer in the documentation
+#     and/or other materials provided with the distribution.
+#  3. The name of the author may not be used to endorse or promote products
+#     derived from this software without specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+# EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+# OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+# OF SUCH DAMAGE.
+
+require 'tb'
+require 'optparse'
+require 'enumerator'
+
+def main(argv)
+  subcommand = argv.shift
+  case subcommand
+  when 'help', '-h' then main_help(argv)
+  when 'csv' then main_csv(argv)
+  when 'tsv' then main_tsv(argv)
+  when 'json' then main_json(argv)
+  when 'yaml' then main_yaml(argv)
+  when 'pp' then main_pp(argv)
+  when 'grep' then main_grep(argv)
+  when 'gsub' then main_gsub(argv)
+  when 'sort' then main_sort(argv)
+  when 'select' then main_select(argv)
+  when 'rename' then main_rename(argv)
+  when 'newfield' then main_newfield(argv)
+  when 'cat' then main_cat(argv)
+  when 'join' then main_join(argv)
+  when 'group' then main_group(argv)
+  when 'cross' then main_cross(argv)
+  when 'shape' then main_shape(argv)
+  when 'mheader' then main_mheader(argv)
+  when 'crop' then main_crop(argv)
+  when nil
+    err "Usage: tb subcommand args..."
+  else
+    err "unexpected subcommand: #{subcommand.inspect}"
+  end
+end
+
+def usage(status)
+  print <<'End'
+Usage:
+  tb csv [OPTS] [TABLE]
+  tb tsv [OPTS] [TABLE]
+  tb json [OPTS] [TABLE]
+  tb yaml [OPTS] [TABLE]
+  tb pp [OPTS] [TABLE]
+  tb grep [OPTS] REGEXP [TABLE]
+  tb gsub [OPTS] REGEXP STRING [TABLE]
+  tb sort [OPTS] [TABLE]
+  tb select [OPTS] FIELD,... [TABLE]
+  tb rename [OPTS] SRC,DST,... [TABLE]
+  tb newfield [OPTS] FIELD RUBY-EXP [TABLE]
+  tb cat [OPTS] [TABLE ...]
+  tb join [OPTS] [TABLE ...]
+  tb group [OPTS] [TABLE]
+  tb cross [OPTS] [TABLE]
+  tb shape [OPTS] [TABLE ...]
+  tb mheader [OPTS] [TABLE]
+  tb crop [OPTS] [TABLE]
+End
+  exit status
+end
+
+def main_help(argv)
+  subcommand = argv.shift
+  case subcommand
+  when 'csv' then puts op_csv
+  when 'tsv' then puts op_tsv
+  when 'json' then puts op_json
+  when 'yaml' then puts op_yaml
+  when 'pp' then puts op_pp
+  when 'grep' then puts op_grep
+  when 'gsub' then puts op_gsub
+  when 'sort' then puts op_sort
+  when 'select' then puts op_select
+  when 'rename' then puts op_rename
+  when 'newfield' then puts op_newfield
+  when 'cat' then puts op_cat
+  when 'join' then puts op_join
+  when 'group' then puts op_group
+  when 'cross' then puts op_cross
+  when 'shape' then puts op_shape
+  when 'mheader' then puts op_mheader
+  when 'crop' then puts op_crop
+  when nil
+    usage(true)
+  else
+    err "unexpected subcommand: #{subcommand.inspect}"
+  end
+end
+
+$opt_N = nil
+$opt_debug = 0
+$opt_no_pager = nil
+
+def op_csv
+  op = OptionParser.new
+  op.banner = 'Usage: tb csv [OPTS] [TABLE]'
+  op.def_option('-h', 'show help message') { puts op; exit 0 }
+  op.def_option('-N', 'use numeric field name') { $opt_N = true }
+  op.def_option('--no-pager', 'don\'t use pager') { $opt_no_pager = true }
+  op
+end
+
+def op_tsv
+  op = OptionParser.new
+  op.banner = 'Usage: tb tsv [OPTS] [TABLE]'
+  op.def_option('-h', 'show help message') { puts op; exit 0 }
+  op.def_option('-N', 'use numeric field name') { $opt_N = true }
+  op.def_option('--no-pager', 'don\'t use pager') { $opt_no_pager = true }
+  op
+end
+
+def op_json
+  op = OptionParser.new
+  op.banner = 'Usage: tb json [OPTS] [TABLE]'
+  op.def_option('-h', 'show help message') { puts op; exit 0 }
+  op.def_option('-N', 'use numeric field name') { $opt_N = true }
+  op.def_option('--no-pager', 'don\'t use pager') { $opt_no_pager = true }
+  op
+end
+
+def op_yaml
+  op = OptionParser.new
+  op.banner = 'Usage: tb yaml [OPTS] [TABLE]'
+  op.def_option('-h', 'show help message') { puts op; exit 0 }
+  op.def_option('-N', 'use numeric field name') { $opt_N = true }
+  op.def_option('--no-pager', 'don\'t use pager') { $opt_no_pager = true }
+  op
+end
+
+def op_pp
+  op = OptionParser.new
+  op.banner = 'Usage: tb pp [OPTS] [TABLE]'
+  op.def_option('-h', 'show help message') { puts op; exit 0 }
+  op.def_option('-N', 'use numeric field name') { $opt_N = true }
+  op.def_option('--no-pager', 'don\'t use pager') { $opt_no_pager = true }
+  op
+end
+
+$opt_grep_e = nil
+$opt_grep_ruby = nil
+$opt_grep_f = nil
+$opt_grep_v = nil
+def op_grep
+  op = OptionParser.new
+  op.banner = 'Usage: tb grep [OPTS] REGEXP [TABLE]'
+  op.def_option('-h', 'show help message') { puts op; exit 0 }
+  op.def_option('-N', 'use numeric field name') { $opt_N = true }
+  op.def_option('-f FIELD', 'search field') {|field| $opt_grep_f = field }
+  op.def_option('-e REGEXP', 'predicate written in ruby.  A hash is given as _.  no usual regexp argument.') {|pattern| $opt_grep_e = pattern }
+  op.def_option('--ruby RUBY-EXP', 'specify a regexp.  no usual regexp argument.') {|ruby_exp| $opt_grep_ruby = ruby_exp }
+  op.def_option('-v', 'ouput the records which doesn\'t match') { $opt_grep_v = true }
+  op.def_option('--no-pager', 'don\'t use pager') { $opt_no_pager = true }
+  op
+end
+
+$opt_gsub_e = nil
+$opt_gsub_f = nil
+def op_gsub
+  op = OptionParser.new
+  op.banner = 'Usage: tb gsub [OPTS] REGEXP STRING [TABLE]'
+  op.def_option('-h', 'show help message') { puts op; exit 0 }
+  op.def_option('-N', 'use numeric field name') { $opt_N = true }
+  op.def_option('-f FIELD', 'search field') {|field| $opt_gsub_f = field }
+  op.def_option('-e REGEXP', 'predicate written in ruby.  A hash is given as _.  no usual regexp argument.') {|pattern| $opt_gsub_e = pattern }
+  op.def_option('--no-pager', 'don\'t use pager') { $opt_no_pager = true }
+  op
+end
+
+$opt_sort_f = nil
+def op_sort
+  op = OptionParser.new
+  op.banner = 'Usage: tb sort [OPTS] [TABLE]'
+  op.def_option('-h', 'show help message') { puts op; exit 0 }
+  op.def_option('-N', 'use numeric field name') { $opt_N = true }
+  op.def_option('-f FIELD,...', 'specify sort keys') {|fs| $opt_sort_f = fs }
+  op.def_option('--no-pager', 'don\'t use pager') { $opt_no_pager = true }
+  op
+end
+
+$opt_select_v = nil
+def op_select
+  op = OptionParser.new
+  op.banner = 'Usage: tb select [OPTS] FIELD,... [TABLE]'
+  op.def_option('-h', 'show help message') { puts op; exit 0 }
+  op.def_option('-N', 'use numeric field name') { $opt_N = true }
+  op.def_option('-v', 'invert match') { $opt_select_v = true }
+  op.def_option('--no-pager', 'don\'t use pager') { $opt_no_pager = true }
+  op
+end
+
+def op_rename
+  op = OptionParser.new
+  op.banner = 'Usage: tb rename [OPTS] SRC,DST,... [TABLE]'
+  op.def_option('-h', 'show help message') { puts op; exit 0 }
+  op.def_option('--no-pager', 'don\'t use pager') { $opt_no_pager = true }
+  op
+end
+
+def op_newfield
+  op = OptionParser.new
+  op.banner = 'Usage: tb newfield [OPTS] FIELD RUBY-EXP [TABLE]'
+  op.def_option('-h', 'show help message') { puts op; exit 0 }
+  op.def_option('--no-pager', 'don\'t use pager') { $opt_no_pager = true }
+  op
+end
+
+def op_cat
+  op = OptionParser.new
+  op.banner = 'Usage: tb cat [OPTS] [TABLE ...]'
+  op.def_option('-h', 'show help message') { puts op; exit 0 }
+  op.def_option('-N', 'use numeric field name') { $opt_N = true }
+  op.def_option('--no-pager', 'don\'t use pager') { $opt_no_pager = true }
+  op
+end
+
+$opt_join_outer = nil
+$opt_join_outer_missing = nil
+def op_join
+  op = OptionParser.new
+  op.banner = 'Usage: tb join [OPTS] [TABLE ...]'
+  op.def_option('-h', 'show help message') { puts op; exit 0 }
+  op.def_option('-d', '--debug', 'show debug message') { $opt_debug += 1 }
+  op.def_option('-N', 'use numeric field name') { $opt_N = true }
+  op.def_option('--outer', 'outer join') { $opt_join_outer = :full }
+  op.def_option('--left', 'left outer join') { $opt_join_outer = :left }
+  op.def_option('--right', 'right outer join') { $opt_join_outer = :right }
+  op.def_option('--outer-missing=DEFAULT', 'missing value for outer join') {|missing|
+    $opt_join_outer ||= :full
+    $opt_join_outer_missing = missing
+  }
+  op.def_option('--no-pager', 'don\'t use pager') { $opt_no_pager = true }
+  op
+end
+
+$opt_group_fields = []
+def op_group
+  op = OptionParser.new
+  op.banner = 'Usage: tb group [OPTS] KEY-FIELD1,... [TABLE]'
+  op.def_option('-h', 'show help message') { puts op; exit 0 }
+  op.def_option('-a AGGREGATION-SPEC[,NEW-FIELD]',
+                '--aggregate AGGREGATION-SPEC[,NEW-FIELD]') {|arg| $opt_group_fields << arg }
+  op.def_option('--no-pager', 'don\'t use pager') { $opt_no_pager = true }
+  op
+end
+
+$opt_cross_fields = []
+$opt_cross_compact = false
+def op_cross
+  op = OptionParser.new
+  op.banner = 'Usage: tb cross [OPTS] HKEY-FIELD1,... VKEY-FIELD1,... [TABLE]'
+  op.def_option('-h', 'show help message') { puts op; exit 0 }
+  op.def_option('-a AGGREGATION-SPEC[,NEW-FIELD]',
+                '--aggregate AGGREGATION-SPEC[,NEW-FIELD]') {|arg| $opt_cross_fields << arg }
+  op.def_option('-c', '--compact', 'compact format') { $opt_cross_compact = true }
+  op.def_option('--no-pager', 'don\'t use pager') { $opt_no_pager = true }
+  op
+end
+
+def op_shape
+  op = OptionParser.new
+  op.banner = 'Usage: tb shape [OPTS] [TABLE ...]'
+  op.def_option('-h', 'show help message') { puts op; exit 0 }
+  op.def_option('-N', 'use numeric field name') { $opt_N = true }
+  op.def_option('--no-pager', 'don\'t use pager') { $opt_no_pager = true }
+  op
+end
+
+$opt_mheader_count = nil
+def op_mheader
+  op = OptionParser.new
+  op.banner = 'Usage: tb mheader [OPTS] [TABLE]'
+  op.def_option('-h', 'show help message') { puts op; exit 0 }
+  op.def_option('-c N', 'number of header records') {|arg| $opt_mheader_count = arg.to_i }
+  op.def_option('--no-pager', 'don\'t use pager') { $opt_no_pager = true }
+  op
+end
+
+$opt_crop_range = nil
+def op_crop
+  op = OptionParser.new
+  op.banner = 'Usage: tb crop [OPTS] [TABLE]'
+  op.def_option('-h', 'show help message') { puts op; exit 0 }
+  op.def_option('-r RANGE', 'range.  i.e. "2,1-4,3", "B1:D3"') {|arg| $opt_crop_range = arg }
+  op.def_option('--no-pager', 'don\'t use pager') { $opt_no_pager = true }
+  op
+end
+
+def err(msg)
+  STDERR.puts msg
+  exit 1
+end
+
+def comparison_value(v)
+  case v
+  when nil
+    []
+  when Numeric
+    [0, v]
+  when String
+    case v
+    when /\A\s*-?\d+\s*\z/
+      [0, Integer(v)]
+    when /\A\s*-?(\d+(\.\d*)?|\.\d+)([eE][-+]?\d+)?\s*\z/
+      [0, Float(v)]
+    else
+      a = []
+      v.scan(/(\d+)|\D+/) {
+        if $1
+          a << 0 << $1.to_i
+        else
+          a << 1 << $&
+        end
+      }
+      a
+    end
+  else
+    raise ArgumentError, "unexpected: #{v.inspect}"
+  end
+end
+
+def conv_to_numeric(v)
+  v = v.strip
+  if /\A-?\d+\z/ =~ v
+    v = v.to_i
+  elsif /\A-?(\d+(\.\d*)?|\.\d+)([eE][-+]?\d+)?\z/ =~ v
+    v = v.to_f
+  else
+    raise "numeric value expected: #{v.inspect}"
+  end
+  v
+end
+
+class CountAggregator
+  def initialize() @result = 0 end
+  def update(v) @result += 1 end
+  def finish() @result end
+end
+
+class SumAggregator
+  def initialize() @result = 0 end 
+  def update(v) @result += conv_to_numeric(v) if !(v.nil? || v == '') end
+  def finish() @result end
+end
+
+class AvgAggregator
+  def initialize() @sum = 0; @count = 0 end 
+  def update(v) @count += 1; @sum += conv_to_numeric(v) if !(v.nil? || v == '') end
+  def finish() @sum / @count.to_f end
+end
+
+class MaxAggregator
+  def initialize() @v = nil; @cmp = nil end 
+  def update(v)
+    cmp = comparison_value(v)
+    if @cmp == nil
+      @v, @cmp = v, cmp
+    else
+      @v, @cmp = v, cmp if (@cmp <=> cmp) < 0
+    end
+  end
+  def finish() @v end
+end
+
+class MinAggregator
+  def initialize() @v = @cmp = nil end 
+  def update(v)
+    cmp = comparison_value(v)
+    if @cmp == nil
+      @v, @cmp = v, cmp
+    else
+      @v, @cmp = v, cmp if (@cmp <=> cmp) > 0
+    end
+  end
+  def finish() @v end
+end
+
+class ValuesAggregator
+  def initialize() @result = [] end 
+  def update(v) @result << v if v end
+  def finish() @result.join(",") end
+end
+
+class UniqueValuesAggregator
+  def initialize() @result = [] end 
+  def update(v) @result << v if v end
+  def finish() @result.uniq.join(",") end
+end
+
+class Selector
+  def initialize(i, aggregator) @i = i; @agg = aggregator end
+  def update(ary) @agg.update(ary[@i]) end
+  def finish() @agg.finish end
+end
+
+def make_aggregator(spec, fs)
+  case spec
+  when 'count'
+    CountAggregator.new
+  when /\Asum\((.*)\)\z/
+    field = $1
+    i = fs.index(field)
+    raise ArgumentError, "field not found: #{field.inspect}" if !i
+    Selector.new(i, SumAggregator.new)
+  when /\Aavg\((.*)\)\z/
+    field = $1
+    i = fs.index(field)
+    raise ArgumentError, "field not found: #{field.inspect}" if !i
+    Selector.new(i, AvgAggregator.new)
+  when /\Amax\((.*)\)\z/
+    field = $1
+    i = fs.index(field)
+    raise ArgumentError, "field not found: #{field.inspect}" if !i
+    Selector.new(i, MaxAggregator.new)
+  when /\Amin\((.*)\)\z/
+    field = $1
+    i = fs.index(field)
+    raise ArgumentError, "field not found: #{field.inspect}" if !i
+    Selector.new(i, MinAggregator.new)
+  when /\Avalues\((.*)\)\z/
+    field = $1
+    i = fs.index(field)
+    raise ArgumentError, "field not found: #{field.inspect}" if !i
+    Selector.new(i, ValuesAggregator.new)
+  when /\Auniquevalues\((.*)\)\z/
+    field = $1
+    i = fs.index(field)
+    raise ArgumentError, "field not found: #{field.inspect}" if !i
+    Selector.new(i, UniqueValuesAggregator.new)
+  else
+    raise ArgumentError, "unexpected aggregation spec: #{spec.inspect}"
+  end
+end
+
+def aggregate(spec, table)
+  update, finish = make_aggregator(spec, table.list_fields)
+  table.each {|rec|
+    update.call(rec.values_at(*fs))
+  }
+  finish.call
+end
+
+def main_csv(argv)
+  op_csv.parse!(argv)
+  each_table_file(argv) {|tbl|
+    with_output {|out|
+      tbl_generate_csv(tbl, out)
+    }
+  }
+end
+
+def main_tsv(argv)
+  op_tsv.parse!(argv)
+  each_table_file(argv) {|tbl|
+    with_output {|out|
+      tbl_generate_tsv(tbl, out)
+    }
+  }
+end
+
+def main_json(argv)
+  require 'json'
+  op_json.parse!(argv)
+  argv = ['-'] if argv.empty?
+  with_output {|out|
+    out.print "["
+    sep = nil
+    argv.each {|filename|
+      sep = ",\n\n" if sep
+      tablereader_open(filename) {|tblreader|
+        tblreader.each {|ary|
+          out.print sep if sep
+          header = tblreader.header
+          h = {}
+          ary.each_with_index {|e, i|
+            h[header[i]] = e if !e.nil?
+          }
+          out.print JSON.pretty_generate(h)
+          sep = ",\n"
+        }
+      }
+    }
+    out.puts "]"
+  }
+end
+
+def main_yaml(argv)
+  require 'yaml'
+  op_yaml.parse!(argv)
+  each_table_file(argv) {|tbl|
+    ary = tbl.map {|rec| rec.to_h }
+    with_output {|out|
+      YAML.dump(ary, out)
+      out.puts
+    }
+  }
+end
+
+def main_pp(argv)
+  op_pp.parse!(argv)
+  argv.unshift '-' if argv.empty?
+  with_output {|out|
+    argv.each {|filename|
+      tablereader_open(filename) {|tblreader|
+        tblreader.each {|ary|
+          h = {}
+          ary.each_with_index {|v, i|
+            next if v.nil?
+            h[tblreader.field_from_index_ex(i)] = v
+          }
+          PP.pp h, out
+        }
+      }
+    }
+  }
+end
+
+def main_grep(argv)
+  op_grep.parse!(argv)
+  if $opt_grep_ruby
+    pred = eval("lambda {|_| #{$opt_grep_ruby} }")
+  elsif $opt_grep_e
+    re = Regexp.new($opt_grep_e)
+    pred = $opt_grep_f ? lambda {|_| re =~ _[$opt_grep_f] } :
+                         lambda {|_| _.any? {|k, v| re =~ v.to_s } }
+  else
+    re = Regexp.new(argv.shift)
+    pred = $opt_grep_f ? lambda {|_| re =~ _[$opt_grep_f] } :
+                         lambda {|_| _.any? {|k, v| re =~ v.to_s } }
+  end
+  opt_v = $opt_grep_v ? true : false
+  argv.unshift '-' if argv.empty?
+  argv.each {|filename|
+    tablereader_open(filename) {|tblreader|
+      with_table_stream_output {|gen|
+        gen.output_header tblreader.header
+        tblreader.each {|ary|
+          h = {}
+          ary.each_with_index {|str, i|
+            f = tblreader.field_from_index_ex(i)
+            h[f] = str
+          }
+          found = pred.call(h)
+          found = opt_v ^ !!(found)
+          gen << ary if found
+        }
+      }
+    }
+  }
+end
+
+def main_gsub(argv)
+  op_gsub.parse!(argv)
+  if $opt_gsub_e
+    re = Regexp.new($opt_gsub_e)
+  else
+    re = Regexp.new(argv.shift)
+  end
+  repl = argv.shift
+  filename = argv.empty? ? '-' : argv.shift
+  warn "extra arguments: #{argv.join(" ")}" if !argv.empty?
+  tablereader_open(filename) {|tblreader|
+    with_table_stream_output {|gen|
+      gen.output_header tblreader.header
+      tblreader.each {|ary|
+        if $opt_gsub_f
+	  ary2 = []
+          ary.each_with_index {|str, i|
+            f = tblreader.field_from_index_ex(i)
+	    if f == $opt_gsub_f
+	      str ||= ''
+	      ary2 << str.gsub(re, repl)
+	    else
+	      ary2 << str
+	    end
+          }
+	else
+	  ary2 = ary.map {|s|
+	    s ||= ''
+	    s.gsub(re, repl)
+	  }
+	end
+	gen << ary2
+      }
+    }
+  }
+end
+
+def main_sort(argv)
+  op_sort.parse!(argv)
+  filename = argv.empty? ? '-' : argv.shift
+  warn "extra arguments: #{argv.join(" ")}" if !argv.empty?
+  if $opt_sort_f
+    fs = split_field_list_argument($opt_sort_f)
+  else
+    fs = nil
+  end
+  tbl = load_table(filename)
+  if fs
+    blk = lambda {|rec| fs.map {|f| comparison_value(rec[f]) } }
+  else
+    blk = lambda {|rec| rec.map {|k, v| comparison_value(v) } }
+  end
+  tbl2 = tbl.reorder_records_by(&blk)
+  with_output {|out|
+    tbl_generate_csv(tbl2, out)
+  }
+end
+
+def main_select(argv)
+  op_select.parse!(argv)
+  fs = split_field_list_argument(argv.shift)
+  filename = argv.shift || '-'
+  warn "extra arguments: #{argv.join(" ")}" if !argv.empty?
+  tablereader_open(filename) {|tblreader|
+    if $opt_select_v
+      h = {}
+      fs.each {|f| h[tblreader.index_from_field(f)] = true }
+      header = nil
+      if !$opt_N
+        header = []
+        tblreader.header.each_with_index {|f, i|
+          header << f if !h[i]
+        }
+      end
+      with_table_stream_output {|gen|
+        gen.output_header(header)
+        tblreader.each {|ary|
+          values = []
+          ary.each_with_index {|v, i|
+            values << v if !h[i]
+          }
+          gen << values
+        }
+      }
+    else
+      header = tblreader.header
+      is = []
+      is = fs.map {|f| tblreader.index_from_field(f) }
+      with_table_stream_output {|gen|
+        gen.output_header(is.map {|i| tblreader.field_from_index_ex(i) })
+        tblreader.each {|ary|
+          gen << ary.values_at(*is)
+        }
+      }
+    end
+  }
+end
+
+def main_rename(argv)
+  op_rename.parse!(argv)
+  fs = split_field_list_argument(argv.shift)
+  filename = argv.shift || '-'
+  warn "extra arguments: #{argv.join(" ")}" if !argv.empty?
+  h = {}
+  fs.each_slice(2) {|sf, df| h[sf] = df }
+  tablereader_open(filename) {|tblreader|
+    header = tblreader.header
+    h.each {|sf, df|
+      unless header.include? sf
+        raise "field not defined: #{sf.inspect}"
+      end
+    }
+    renamed_header = tblreader.header.map {|f| h.fetch(f, f) }
+    with_table_stream_output {|gen|
+      gen.output_header(renamed_header)
+      tblreader.each {|ary|
+        gen << ary
+      }
+    }
+  }
+end
+
+def main_newfield(argv)
+  op_rename.parse!(argv)
+  field = argv.shift
+  rubyexp = argv.shift
+  pr = eval("lambda {|_| #{rubyexp} }")
+  filename = argv.shift || '-'
+  warn "extra arguments: #{argv.join(" ")}" if !argv.empty?
+  tablereader_open(filename) {|tblreader|
+    renamed_header = [field] + tblreader.header
+    with_table_stream_output {|gen|
+      gen.output_header(renamed_header)
+      tblreader.each {|ary|
+        h = {}
+        ary.each_with_index {|str, i|
+          f = tblreader.field_from_index_ex(i)
+          h[f] = str
+        }
+        gen << [pr.call(h), *ary]
+      }
+    }
+  }
+end
+
+def main_cat(argv)
+  op_cat.parse!(argv)
+  argv = ['-'] if argv.empty?
+  if $opt_N
+    argv.each {|filename|
+      with_table_stream_output {|gen|
+        tablereader_open(filename) {|tblreader|
+          tblreader.each {|ary|
+            gen << ary
+          }
+        }
+      }
+    }
+  else
+    readers = []
+    h = {}
+    argv.each {|filename|
+      r = tablereader_open(filename)
+      readers << r
+      r.header.each {|f|
+        h[f] = h.size if !h[f]
+      }
+    }
+    with_table_stream_output {|gen|
+      gen.output_header h.keys.sort_by {|k| h[k] }
+      readers.each {|r|
+        header = r.header.dup
+        r.each {|ary|
+          while header.length < ary.length
+            f = r.field_from_index_ex(header.length)
+            header << f
+            h[f] = h.size if !h[f]
+          end
+          ary2 = []
+          ary.each_with_index {|v, i|
+            f = r.field_from_index(i)
+            j = h.fetch(f)
+            ary2[j] = v
+          }
+          gen << ary2
+        }
+      }
+    }
+  end
+end
+
+def main_join(argv)
+  op_join.parse!(argv)
+  result = Tb.new([], [])
+  retain_left = false
+  retain_right = false
+  case $opt_join_outer
+  when :full
+    retain_left = true
+    retain_right = true
+  when :left
+    retain_left = true
+  when :right
+    retain_right = true
+  when nil
+  else
+    raise "unexpected $opt_join_outer: #{$opt_join_outer.inspect}"
+  end
+  if $opt_join_outer
+    each_table_file(argv) {|tbl|
+      STDERR.puts "shared keys: #{(result.list_fields & tbl.list_fields).inspect}" if 1 <= $opt_debug
+      result = result.natjoin2_outer(tbl, $opt_join_outer_missing, retain_left, retain_right)
+    }
+  else
+    each_table_file(argv) {|tbl|
+      STDERR.puts "shared keys: #{(result.list_fields & tbl.list_fields).inspect}" if 1 <= $opt_debug
+      result = result.natjoin2(tbl)
+    }
+  end
+  with_output {|out|
+    tbl_generate_csv(result, out)
+  }
+end
+
+def main_group(argv)
+  op_group.parse!(argv)
+  kfs = split_field_list_argument(argv.shift)
+  opt_group_fields = $opt_group_fields.map {|arg|
+    aggregation_spec, new_field = split_field_list_argument(arg)
+    new_field ||= aggregation_spec
+    [new_field, lambda {|fields| make_aggregator(aggregation_spec, fields) } ]
+  }
+  filename = argv.shift || '-'
+  warn "extra arguments: #{argv.join(" ")}" if !argv.empty?
+  h = {}
+  tablereader_open(filename) {|tblreader|
+    kis = kfs.map {|f| tblreader.index_from_field(f) }
+    result_fields = kfs + opt_group_fields.map {|nf, maker| nf }
+    tblreader.each {|ary|
+      kvs = ary.values_at(*kis)
+      if !h.include?(kvs)
+        h[kvs] = opt_group_fields.map {|nf, maker| ag = maker.call(tblreader.header); ag.update(ary); ag }
+      else
+        h[kvs].each {|ag|
+          ag.update(ary)
+        }
+      end
+    }
+    result = Tb.new(result_fields)
+    h.keys.sort_by {|k| k.map {|v| comparison_value(v) } }.each {|k|
+      a = h[k]
+      result.insert_values result_fields, k + a.map {|ag| ag.finish }
+    }
+    with_output {|out|
+      tbl_generate_csv(result, out)
+    }
+  }
+end
+
+def main_cross(argv)
+  op_cross.parse!(argv)
+  hkfs = split_field_list_argument(argv.shift)
+  vkfs = split_field_list_argument(argv.shift)
+  if $opt_cross_fields.empty?
+    opt_cross_fields = [['count', 'count']]
+  else
+    opt_cross_fields = $opt_cross_fields.map {|arg|
+      agg_spec, new_field = split_field_list_argument(arg)
+      new_field ||= agg_spec
+      [agg_spec, new_field]
+    }
+  end
+  filename = argv.shift || '-'
+  warn "extra arguments: #{argv.join(" ")}" if !argv.empty?
+  tablereader_open(filename) {|tblreader|
+    vkis = vkfs.map {|f| tblreader.index_from_field(f) }
+    hkis = hkfs.map {|f| tblreader.index_from_field(f) }
+    vset = {}
+    hset = {}
+    set = {}
+    tblreader.each {|ary|
+      vkvs = ary.values_at(*vkis)
+      hkvs = ary.values_at(*hkis)
+      vset[vkvs] = true if !vset.include?(vkvs)
+      hset[hkvs] = true if !hset.include?(hkvs)
+      if !set.include?([vkvs, hkvs])
+        set[[vkvs, hkvs]] = opt_cross_fields.map {|agg_spec, nf|
+          ag = make_aggregator(agg_spec, tblreader.header)
+          ag.update(ary)
+          ag
+        }
+      else
+        set[[vkvs, hkvs]].each {|ag|
+          ag.update(ary)
+        }
+      end
+    }
+    vary = vset.keys.sort_by {|a| a.map {|v| comparison_value(v) } }
+    hary = hset.keys.sort_by {|a| a.map {|v| comparison_value(v) } }
+    with_output {|out|
+      Tb.csv_stream_output(out) {|gen|
+        hkfs.each_with_index {|hkf, i|
+          next if $opt_cross_compact && i == hkfs.length - 1
+          row = [nil] * (vkfs.length - 1) + [hkf]
+          hary.each {|hkvs| opt_cross_fields.length.times { row << hkvs[i] } }
+          gen << row
+        }
+        if $opt_cross_compact
+          r = vkfs.dup
+          hary.each {|hkvs| r.concat([hkvs[-1]] * opt_cross_fields.length) }
+          gen << r
+        else
+          r = vkfs.dup
+          hary.each {|hkvs| r.concat opt_cross_fields.map {|agg_spec, new_field| new_field } }
+          gen << r
+        end
+        vary.each {|vkvs|
+          row = vkvs.dup
+          hary.each {|hkvs|
+            ags = set[[vkvs, hkvs]]
+            if !ags
+              opt_cross_fields.length.times { row << nil }
+            else
+              ags.each {|ag| row << ag.finish }
+            end
+          }
+          gen << row
+        }
+      }
+    }
+  }
+end
+
+def main_shape(argv)
+  op_shape.parse!(argv)
+  filenames = argv.empty? ? ['-'] : argv
+  result = Tb.new(%w[header_fields min_fields max_fields records filename])
+  filenames.each {|filename|
+    tablereader_open(filename) {|tblreader|
+      num_header_fields = tblreader.header.length
+      min_num_fields = nil
+      max_num_fields = nil
+      num_records = 0
+      tblreader.each {|ary|
+        num_records += 1
+        n = ary.length
+        if min_num_fields.nil?
+          min_num_fields = max_num_fields = n
+        else
+          min_num_fields = n if n < min_num_fields
+          max_num_fields = n if max_num_fields < n
+        end
+      }
+      result.insert({'header_fields'=>num_header_fields,
+                     'min_fields'=>min_num_fields,
+                     'max_fields'=>max_num_fields,
+                     'records'=>num_records,
+                     'filename'=>filename})
+    }
+  }
+  with_output {|out|
+    # don't use tbl_generate_csv() because the header should always outputted.
+    result.generate_csv(out)
+  }
+end
+
+def main_mheader(argv)
+  op_mheader.parse!(argv)
+  filename = argv.shift || '-'
+  warn "extra arguments: #{argv.join(" ")}" if !argv.empty?
+  header = []
+  if $opt_mheader_count
+    c = $opt_mheader_count
+    header_end_p = lambda {
+      c -= 1
+      c == 0 ? header.map {|a| a.compact.join(' ').strip } : nil
+    }
+  else
+    header_end_p = lambda {
+      h2 = header.map {|a| a.compact.join(' ').strip }.uniq
+      header.length == h2.length ? h2 : nil
+    }
+  end
+  with_table_stream_output {|gen|
+    Tb::Reader.open(filename, {:numeric=>true}) {|tblreader|
+      tblreader.each {|ary|
+        if header
+          ary.each_with_index {|v,i|
+            header[i] ||= []
+            header[i] << v if header[i].empty? || header[i].last != v
+          }
+          h2 = header_end_p.call
+          if h2
+            gen << h2
+            header = nil
+          end
+        else
+          gen << ary
+        end
+      }
+    }
+  }
+  if header
+    warn "no header found."
+  end
+end
+
+def main_crop(argv)
+  op_crop.parse!(argv)
+  filename = argv.shift || '-'
+  warn "extra arguments: #{argv.join(" ")}" if !argv.empty?
+  stream = false
+  if $opt_crop_range
+    case $opt_crop_range
+    when /\A(\d+),(\d+)-(\d+),(\d+)\z/ # 1-based
+      stream = true
+      range_col1 = $1.to_i
+      range_row1 = $2.to_i
+      range_col2 = $3.to_i
+      range_row2 = $4.to_i
+    when /\A([A-Z]+)(\d+):([A-Z]+)(\d+)\z/ # 1-based
+      stream = true
+      range_col1 = decode_a1_addressing_col($1)
+      range_row1 = $2.to_i
+      range_col2 = decode_a1_addressing_col($3)
+      range_row2 = $4.to_i
+    else
+      raise ArgumentError, "unexpected range argument: #{$opt_crop_range.inspect}"
+    end
+  end
+  if stream
+    with_table_stream_output {|gen|
+      Tb::Reader.open(filename, {:numeric=>true}) {|tblreader|
+        rownum = 1
+	tblreader.each {|ary|
+	  if range_row2 < rownum
+	    break
+	  end
+	  if range_row1 <= rownum
+	    if range_col2 < ary.length
+	      ary[range_col2..-1] = []
+	    end
+	    if 1 < range_col1
+	      ary[0...(range_col1-1)] = []
+	    end
+	    gen << ary
+	  end
+	  rownum += 1
+	}
+      }
+    }
+  else
+    arys = []
+    Tb::Reader.open(filename, {:numeric=>true}) {|tblreader|
+      tblreader.each {|a|
+        a.pop while !a.empty? && (a.last.nil? || a.last == '')
+	arys << a
+      }
+    }
+    arys.pop while !arys.empty? && arys.last.all? {|v| v.nil? || v == '' }
+    arys.shift while !arys.empty? && arys.first.all? {|v| v.nil? || v == '' }
+    if !arys.empty?
+      while arys.all? {|a| a.empty? || (a.first.nil? || a.first == '') }
+	arys.each {|a| a.shift }
+      end
+    end
+    with_table_stream_output {|gen|
+      arys.each {|a| gen << a }
+    }
+  end
+end
+
+def decode_a1_addressing_col(str)
+  (26**str.length-1)/25+str.tr("A-Z", "0-9A-P").to_i(26)
+end
+
+def split_field_list_argument(arg)
+  split_csv_argument(arg).map {|f| f || '' }
+end
+
+def split_csv_argument(arg)
+  Tb.csv_stream_input(arg) {|ary| return ary }
+  return []
+end
+
+def each_table_file(argv)
+  if argv.empty?
+    yield load_table('-')
+  else
+    argv.each {|filename|
+      tbl = load_table(filename)
+      yield tbl
+    }
+  end
+end
+
+def load_table(filename)
+  tablereader_open(filename) {|tblreader|
+    arys = []
+    tblreader.each {|ary|
+      arys << ary
+    }
+    header = tblreader.header
+    tbl = Tb.new(header)
+    arys.each {|ary|
+      ary << nil while ary.length < header.length
+      tbl.insert_values header, ary
+    }
+    tbl
+  }
+end
+
+def tablereader_open(filename, &b)
+  Tb::Reader.open(filename, {:numeric=>$opt_N}, &b)
+end
+
+def with_table_stream_output
+  with_output {|out|
+    Tb.csv_stream_output(out) {|gen|
+      def gen.output_header(header)
+        self << header if !$opt_N
+      end
+      yield gen
+    }
+  }
+end
+
+def tbl_generate_csv(tbl, out)
+  if $opt_N
+    header = tbl.list_fields
+    Tb.csv_stream_output(out) {|gen|
+      tbl.each {|rec|
+        gen << rec.values_at(*header)
+      }
+    }
+  else
+    tbl.generate_csv(out)
+  end
+end
+
+def tbl_generate_tsv(tbl, out)
+  if $opt_N
+    header = tbl.list_fields
+    Tb.tsv_stream_output(out) {|gen|
+      tbl.each {|rec|
+        gen << rec.values_at(*header)
+      }
+    }
+  else
+    tbl.generate_tsv(out)
+  end
+end
+
+def with_output
+  if STDOUT.tty? && !$opt_no_pager
+    IO.popen(ENV['PAGER'] || 'more', 'w') {|pager|
+      yield pager
+    }
+  else
+    yield STDOUT
+  end
+end
+
+main ARGV