tb 0.3 → 0.4

data/lib/tb/csv.rb

@@ -63,37 +63,12 @@ class Tb
   end
 
   class CSVReader
-    if defined? CSV::Reader
-      # Ruby 1.8
-      def initialize(input)
-        if input.respond_to? :to_str
-          @csv = CSV::StringReader.new(input)
-        else
-          @csv = CSV::IOReader.new(input)
-        end
-        @eof = false
-      end
-
-      def shift
-        return nil if @eof
-        ary = @csv.shift
-        if ary.empty?
-          ary = nil
-          @eof = true
-        elsif ary == [nil]
-          ary = []
-        end
-        ary
-      end
-    else
-      # Ruby 1.9
-      def initialize(input)
-        @csv = CSV.new(input)
-      end
+    def initialize(input)
+      @csv = CSV.new(input)
+    end
 
-      def shift
-        @csv.shift
-      end
+    def shift
+      @csv.shift
     end
 
     def each
@@ -106,35 +81,17 @@ class Tb
 
   def Tb.csv_stream_output(out)
     require 'csv'
-    if defined? CSV::Writer
-      # Ruby 1.8
-      CSV::Writer.generate(out) {|csvgen|
-        yield csvgen
-      }
-    else
-      # Ruby 1.9
-      gen = Object.new
-      gen.instance_variable_set(:@out, out)
-      def gen.<<(ary)
-        @out << ary.to_csv
-      end
-      yield gen
+    gen = Object.new
+    gen.instance_variable_set(:@out, out)
+    def gen.<<(ary)
+      @out << ary.to_csv
     end
+    yield gen
   end
 
   def Tb.csv_encode_row(ary)
     require 'csv'
-    if defined? CSV::Writer
-      # Ruby 1.8
-      out = ''
-      CSV::Writer.generate(out) {|csvgen|
-        csvgen << ary
-      }
-      out
-    else
-      # Ruby 1.9
-      ary.to_csv
-    end
+    ary.to_csv
   end
 
   # :call-seq:

data/lib/tb/customcmp.rb

@@ -0,0 +1,41 @@
+# Copyright (C) 2012 Tanaka Akira  <akr@fsij.org>
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 
+#  1. Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#  2. Redistributions in binary form must reproduce the above
+#     copyright notice, this list of conditions and the following
+#     disclaimer in the documentation and/or other materials provided
+#     with the distribution.
+#  3. The name of the author may not be used to endorse or promote
+#     products derived from this software without specific prior
+#     written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+# GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+class Tb::CustomCmp
+  include Comparable
+
+  def initialize(customcmp_object, &cmp)
+    @customcmp_object = customcmp_object
+    @cmp = cmp
+  end
+  attr_reader :customcmp_object, :cmp
+
+  def <=> other
+    @cmp.call(@customcmp_object, other.customcmp_object)
+  end
+end

data/lib/tb/customeq.rb

@@ -0,0 +1,41 @@
+# Copyright (C) 2012 Tanaka Akira  <akr@fsij.org>
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 
+#  1. Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#  2. Redistributions in binary form must reproduce the above
+#     copyright notice, this list of conditions and the following
+#     disclaimer in the documentation and/or other materials provided
+#     with the distribution.
+#  3. The name of the author may not be used to endorse or promote
+#     products derived from this software without specific prior
+#     written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+# GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+class Tb::CustomEq
+  include Comparable
+
+  def initialize(customeq_object, &eq)
+    @customeq_object = customeq_object
+    @eq = eq
+  end
+  attr_reader :customeq_object, :eq
+
+  def ==(other)
+    @eq.call(@customeq_object, other.customeq_object)
+  end
+end

data/lib/tb/enumerable.rb

@@ -1,474 +1,264 @@
-# lib/tb/enumerable.rb - extensions for Enumerable
-#
-# Copyright (C) 2010-2012 Tanaka Akira  <akr@fsij.org>
-#
+# Copyright (C) 2012 Tanaka Akira  <akr@fsij.org>
+# 
 # Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-#  1. Redistributions of source code must retain the above copyright notice, this
-#     list of conditions and the following disclaimer.
-#  2. Redistributions in binary form must reproduce the above copyright notice,
-#     this list of conditions and the following disclaimer in the documentation
-#     and/or other materials provided with the distribution.
-#  3. The name of the author may not be used to endorse or promote products
-#     derived from this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
-# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
-# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
-# EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
-# OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
-# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
-# OF SUCH DAMAGE.
+# modification, are permitted provided that the following conditions
+# are met:
+# 
+#  1. Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#  2. Redistributions in binary form must reproduce the above
+#     copyright notice, this list of conditions and the following
+#     disclaimer in the documentation and/or other materials provided
+#     with the distribution.
+#  3. The name of the author may not be used to endorse or promote
+#     products derived from this software without specific prior
+#     written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+# GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-module Enumerable
-  # :call-seq:
-  #   enum.tb_categorize(ksel1, ksel2, ..., vsel, [opts])
-  #   enum.tb_categorize(ksel1, ksel2, ..., vsel, [opts]) {|ks, vs| ... }
-  #
-  # categorizes the elements in _enum_ and returns a hash.
-  # This method assumes multiple elements for a category.
-  #
-  # +tb_categorize+ takes one or more key selectors,
-  # one value selector and
-  # an optional option hash.
-  # It also takes an optional block.
-  #
-  # The selectors specify how to extract a value from an element in _enum_.
-  #
-  # The key selectors, _kselN_, are used to extract hash keys from an element.
-  # If two or more key selectors are specified, the result hash will be nested.
-  #
-  # The value selector, _vsel_, is used for the values of innermost hashes.
-  # By default, all values extracted by _vsel_ from the elements which
-  # key selectors extracts same value are composed as an array.
-  # The array is set to the values of the innermost hashes.
-  # This behavior can be customized by the options: :seed, :op and :update.
-  #
-  #   a = [{:fruit => "banana", :color => "yellow", :taste => "sweet", :price => 100},
-  #        {:fruit => "melon", :color => "green", :taste => "sweet", :price => 300},
-  #        {:fruit => "grapefruit", :color => "yellow", :taste => "tart", :price => 200}]
-  #   p a.tb_categorize(:color, :fruit)
-  #   #=> {"yellow"=>["banana", "grapefruit"], "green"=>["melon"]}
-  #   p a.tb_categorize(:taste, :fruit)
-  #   #=> {"sweet"=>["banana", "melon"], "tart"=>["grapefruit"]}
-  #   p a.tb_categorize(:taste, :color, :fruit)
-  #   #=> {"sweet"=>{"yellow"=>["banana"], "green"=>["melon"]}, "tart"=>{"yellow"=>["grapefruit"]}}
-  #   p a.tb_categorize(:taste, :color)
-  #   #=> {"sweet"=>["yellow", "green"], "tart"=>["yellow"]}
-  #
-  # In the above example, :fruit, :color and :taste is specified as selectors.
-  # There are several types of selectors as follows:
-  #
-  # - object with +call+ method (procedure, etc.): extracts a value from the element by calling the procedure with the element as an argument.
-  # - array of selectors: make an array which contains the values extracted by the selectors.
-  # - other object: extracts a value from the element using +[]+ method as +element[selector]+.
-  #
-  # So the selector :fruit extracts the value from the element
-  # {:fruit => "banana", :color => "yellow", :taste => "sweet", :price => 100}
-  # as {...}[:fruit].
-  #
-  #   p a.tb_categorize(lambda {|elt| elt[:fruit][4] }, :fruit)
-  #   #=> {"n"=>["banana", "melon"], "e"=>["grapefruit"]}
-  #
-  # When the key selectors returns same key for two or or more elements,
-  # corresponding values extracted by the value selector are combined.
-  # By default, all values are collected as an array.
-  # :seed, :op and :update option in the option hash customizes this behavior.
-  # :seed option and :op option is similar to Enumerable#inject.
-  # :seed option specifies an initial value.
-  # (If :seed option is not given, the first value for each category is treated as an initial value.)
-  # :op option specifies a procedure to combine a seed and an element into a next seed.
-  # :update option is same as :op option except it takes three arguments instead of two:
-  # keys, seed and element.
-  # +to_proc+ method is used to convert :op and :update option to a procedure.
-  # So a symbol can be used for them.
-  #
-  #   # count categorized elements.
-  #   p a.tb_categorize(:color, lambda {|e| 1 }, :op=>:+)
-  #   #=> {"yellow"=>2, "green"=>1}
-  #
-  #   p a.tb_categorize(:color, :fruit, :seed=>"", :op=>:+)
-  #   #=> {"yellow"=>"bananagrapefruit", "green"=>"melon"}
-  #
-  # The default behavior, collecting all values as an array, is implemented as follows.
-  #   :seed => nil
-  #   :update => {|ks, s, v| !s ? [v] : (s << v) }
-  #
-  # :op and :update option are disjoint.
-  # ArgumentError is raised if both are specified.
-  #
-  # The block for +tb_categorize+ method converts combined values to final innermost hash values.
-  #
-  #   p a.tb_categorize(:color, :fruit) {|ks, vs| vs.join(",") }
-  #   #=> {"yellow"=>"banana,grapefruit", "green"=>"melon"}
-  #
-  #   # calculates the average price for fruits of each color.
-  #   p a.tb_categorize(:color, :price) {|ks, vs| vs.inject(0.0, &:+) / vs.length }
-  #   #=> {"yellow"=>150.0, "green"=>300.0}
-  #
-  def tb_categorize(*args, &reduce_proc)
-    opts = args.last.kind_of?(Hash) ? args.pop : {}
-    if args.length < 2
-      raise ArgumentError, "needs 2 or more arguments without option hash (but #{args.length})"
-    end
-    value_selector = tb_cat_selector_proc(args.pop)
-    key_selectors = args.map {|a| tb_cat_selector_proc(a) }
-    has_seed = opts.has_key? :seed
-    seed_value = opts[:seed]
-    if opts.has_key?(:update) && opts.has_key?(:op)
-      raise ArgumentError, "both :op and :update option specified"
-    elsif opts.has_key? :update
-      update_proc = opts[:update].to_proc
-    elsif opts.has_key? :op
-      op_proc = opts[:op].to_proc
-      update_proc = lambda {|ks, s, v| op_proc.call(s, v) }
-    else
-      has_seed = true
-      seed_value = nil
-      update_proc = lambda {|ks, s, v| !s ? [v] : (s << v) }
-    end
-    result = {}
-    each {|*elts|
-      elt = elts.length <= 1 ? elts[0] : elts
-      ks = key_selectors.map {|ksel| ksel.call(elt) }
-      v = value_selector.call(elt)
-      h = result
-      0.upto(ks.length-2) {|i|
-        k = ks[i]
-        h[k] = {} if !h.has_key?(k)
-        h = h[k]
+module Tb::Enumerable
+  include Enumerable
+
+  def with_header(&header_proc)
+    Enumerator.new {|y|
+      header_and_each(header_proc) {|pairs|
+        y.yield pairs
       }
-      lastk = ks.last
-      if !h.has_key?(lastk)
-        if has_seed
-          h[lastk] = update_proc.call(ks, seed_value, v)
-        else
-          h[lastk] = v
-        end
-      else
-        h[lastk] = update_proc.call(ks, h[lastk], v)
-      end
     }
-    if reduce_proc
-      tb_cat_reduce(result, [], key_selectors.length-1, reduce_proc)
-    end
-    result
   end
 
-  def tb_cat_selector_proc(selector)
-    if selector.respond_to?(:call)
-      selector
-    elsif selector.respond_to? :to_ary
-      selector_procs = selector.to_ary.map {|sel| tb_cat_selector_proc(sel) }
-      lambda {|elt| selector_procs.map {|selproc| selproc.call(elt) } }
-    else
-      lambda {|elt| elt[selector] }
-    end
+  def with_cumulative_header(&header_proc)
+    Enumerator.new {|y|
+      hset = {}
+      internal_header_proc = lambda {|header0|
+        if header0
+          header0.each {|f|
+            hset[f] = true
+          }
+        end
+        header_proc.call(header0) if header_proc
+      }
+      header_and_each(internal_header_proc) {|pairs|
+        pairs.each {|f, v|
+          if !hset[f]
+            hset[f] = true
+          end
+        }
+        y.yield [pairs, hset.keys.freeze]
+      }
+    }
   end
-  private :tb_cat_selector_proc
 
-  def tb_cat_reduce(hash, ks, nestlevel, reduce_proc)
-    if nestlevel.zero?
-      hash.each {|k, v|
-        ks << k
-        begin
-          hash[k] = reduce_proc.call(ks.dup, v)
-        ensure
-          ks.pop
+  def cat(*ers, &b)
+    ers = [self, *ers]
+    rec = lambda {|y, header|
+      if ers.empty?
+        if header
+          y.set_header header
         end
-      }
+      else
+        last_e = ers.pop
+        last_e.with_header {|last_e_header|
+          if last_e_header && header
+            header = last_e_header | header
+          else
+            header = nil
+          end
+          rec.call(y, header)
+        }.each {|v|
+          y.yield v
+        }
+      end
+    }
+    er = Tb::Enumerator.new {|y|
+      rec.call(y, [])
+    }
+    if block_given?
+      er.each(&b)
     else
-      hash.each {|k, h|
-        ks << k
-        begin
-          tb_cat_reduce(h, ks, nestlevel-1, reduce_proc)
-        ensure
-          ks.pop
-        end
-      }
+      er
     end
   end
-  private :tb_cat_reduce
 
-  # :call-seq:
-  #   enum.tb_unique_categorize(ksel1, ksel2, ..., vsel, [opts]) -> hash
-  #   enum.tb_unique_categorize(ksel1, ksel2, ..., vsel, [opts]) {|s, v| ... } -> hash
-  #
-  # categorizes the elements in _enum_ and returns a hash.
-  # This method assumes one element for a category by default.
-  #
-  # +tb_unique_categorize+ takes one or more key selectors,
-  # one value selector and
-  # an optional option hash.
-  # It also takes an optional block.
-  #
-  # The selectors specify how to extract a value from an element in _enum_.
-  # See Enumerable#tb_categorize for details of selectors.
-  #
-  # The key selectors, _kselN_, are used to extract hash keys from an element.
-  # If two or more key selectors are specified, the result hash will be nested.
-  #
-  # The value selector, _vsel_, is used for the values of innermost hashes.
-  # By default, this method assumes the key selectors categorizes elements in enum uniquely.
-  # If the key selectors generates same keys for two or more elements, ArgumentError is raised.
-  # This behavior can be customized by :seed option and the block.
-  #
-  #   a = [{:fruit => "banana", :color => "yellow", :taste => "sweet", :price => 100},
-  #        {:fruit => "melon", :color => "green", :taste => "sweet", :price => 300},
-  #        {:fruit => "grapefruit", :color => "yellow", :taste => "tart", :price => 200}]
-  #   p a.tb_unique_categorize(:fruit, :price)
-  #   #=> {"banana"=>100, "melon"=>300, "grapefruit"=>200}
-  #
-  #   p a.tb_unique_categorize(:color, :price)
-  #   # ArgumentError
-  #
-  # If the block is given, it is used for combining values in a category.
-  # The arguments for the block is a seed and the value extracted by _vsel_.
-  # The return value of the block is used as the next seed.
-  # :seed option specifies the initial seed.
-  # If :seed is not given, the first value for each category is used for the seed.
-  #
-  #   p a.tb_unique_categorize(:taste, :price) {|s, v| s + v }
-  #   #=> {"sweet"=>400, "tart"=>200}
-  #
-  #   p a.tb_unique_categorize(:color, :price) {|s, v| s + v }
-  #   #=> {"yellow"=>300, "green"=>300}
-  #
-  def tb_unique_categorize(*args, &update_proc)
-    opts = args.last.kind_of?(Hash) ? args.pop.dup : {}
-    if update_proc
-      opts[:update] = lambda {|ks, s, v| update_proc.call(s, v) }
-    else
-      seed = Object.new
-      opts[:seed] = seed
-      opts[:update] = lambda {|ks, s, v|
-        if s.equal? seed
-          v
-        else
-          raise ArgumentError, "ambiguous key: #{ks.map {|k| k.inspect }.join(',')}"
+  # creates a new Tb::Enumerator object which have
+  # new field named by _field_ with the value returned by the block.
+  #
+  #   t1 = Tb.new %w[a b], [1, 2], [3, 4]
+  #   p t1.newfield("x") {|row| row["a"] + row["b"] + 100 }.to_a
+  #   #=> [{"x"=>103, "a"=>1, "b"=>2},
+  #   #    {"x"=>107, "a"=>3, "b"=>4}]
+  #
+  def newfield(field)
+    Tb::Enumerator.new {|y|
+      self.with_header {|header|
+        if header
+          y.set_header(Tb::FieldSet.normalize([field, *header]))
         end
+      }.each {|row|
+        keys = row.keys
+        keys = Tb::FieldSet.normalize([field, *keys])
+        vals = row.values
+        vals = [yield(row), *vals]
+        y << Hash[keys.zip(vals)]
       }
-    end
-    tb_categorize(*(args + [opts]))
+    }
   end
 
   # :call-seq:
-  #   enum.tb_category_count(ksel1, ksel2, ...)
-  #
-  # counts elements in _enum_ for each category defined by the key selectors.
-  #
-  #   a = [{:fruit => "banana", :color => "yellow", :taste => "sweet", :price => 100},
-  #        {:fruit => "melon", :color => "green", :taste => "sweet", :price => 300},
-  #        {:fruit => "grapefruit", :color => "yellow", :taste => "tart", :price => 200}]
-  #
-  #   p a.tb_category_count(:color)
-  #   #=> {"yellow"=>2, "green"=>1}
-  #
-  #   p a.tb_category_count(:taste)
-  #   #=> {"sweet"=>2, "tart"=>1}
-  #
-  #   p a.tb_category_count(:taste, :color)
-  #   #=> {"sweet"=>{"yellow"=>1, "green"=>1}, "tart"=>{"yellow"=>1}}
-  #
-  # The selectors specify how to extract a value from an element in _enum_.
-  # See Enumerable#tb_categorize for details of selectors.
-  #
-  def tb_category_count(*args)
-    tb_categorize(*(args + [lambda {|e| 1 }, {:update => lambda {|ks, s, v| s + v }}]))
-  end
-
-  def dump_objsfile(title, tempfile)
-    tempfile.flush
-    path = tempfile
-    a = []
-    open(path) {|f|
-      until f.eof?
-        pair = Marshal.load(f)
-        a << (pair ? pair.last : :sep)
-      end
+  #   table1.natjoin2(table2, missing_value=nil, retain_left=false, retain_right=false)
+  def natjoin2(tbl2, missing_value=nil, retain_left=false, retain_right=false)
+    Tb::Enumerator.new {|y|
+      tbl1 = self
+      header1 = header2 = nil
+      sorted_tbl2 = nil
+      common_header = nil
+      total_header = nil
+      sorted_tbl1 = tbl1.with_header {|h1|
+        header1 = h1
+        sorted_tbl2 = tbl2.with_header {|h2|
+          header2 = h2
+          common_header = header1 & header2
+          total_header = header1 | header2
+          y.set_header total_header
+        }.lazy_map {|pairs|
+          [common_header.map {|f| pairs[f] }, pairs]
+        }.extsort_by {|cv, pairs| cv }.to_fileenumerator
+      }.lazy_map {|pairs|
+        [common_header.map {|f| pairs[f] }, pairs]
+      }.extsort_by {|cv, pairs| cv }.to_fileenumerator
+      sorted_tbl1.open_reader {|t1|
+        sorted_tbl2.open_reader {|t2|
+          missing_hash = {}
+          total_header.each {|f|
+            missing_hash[f] = missing_value
+          }
+          Tb::ExEnumerator.merge_sorted(t1, t2) {|cv, t1_or_nil, t2_or_nil|
+            if !t2_or_nil
+              t1.subeach_by {|_cv1, _| _cv1 }.each {|_, _pairs1|
+                if retain_left
+                  y.yield missing_hash.merge(_pairs1.to_hash)
+                end
+              }
+            elsif !t1_or_nil
+              t2.subeach_by {|_cv2, _| _cv2 }.each {|_, _pairs2|
+                if retain_right
+                  y.yield missing_hash.merge(_pairs2.to_hash)
+                end
+              }
+            else # t1_or_nil && t1_or_nil
+              t2_pos = t2.pos
+              t1.subeach_by {|_cv1, _| _cv1 }.each {|_, _pairs1|
+                t2.pos = t2_pos
+                t2.subeach_by {|_cv2, _| _cv2 }.each {|_, _pairs2|
+                  y.yield(_pairs2.to_hash.merge(_pairs1.to_hash))
+                }
+              }
+            end
+          }
+        }
+      }
     }
-    puts "#{title}: #{a.inspect}"
   end
-  private :dump_objsfile
 
-  def extsort_by(opts={}, &cmpvalue_from)
-    memsize = opts[:memsize] || 10000000
-    Enumerator.new {|y|
-      extsort_by_internal(memsize, cmpvalue_from, y)
-    }
+  # :call-seq:
+  #   table1.natjoin2_outer(table2, missing=nil, retain_left=true, retain_right=true)
+  def natjoin2_outer(tbl2, missing_value=nil, retain_left=true, retain_right=true)
+    natjoin2(tbl2, missing_value, retain_left, retain_right)
   end
 
-  def extsort_by_internal(memsize, cmpvalue_from, y)
-    tmp1 = Tempfile.new("tbsortA")
-    tmp2 = Tempfile.new("tbsortB")
-    extsort_by_first_split(tmp1, tmp2, cmpvalue_from, memsize)
-    if tmp1.size == 0 && tmp2.size == 0
-      return Enumerator.new {|_| }
-    end
-    tmp3 = Tempfile.new("tbsortC")
-    tmp4 = Tempfile.new("tbsortD")
-    while tmp2.size != 0
-      #dump_objsfile(:tmp1, tmp1)
-      #dump_objsfile(:tmp2, tmp2)
-      #dump_objsfile(:tmp3, tmp3)
-      #dump_objsfile(:tmp4, tmp4)
-      extsort_by_merge(tmp1, tmp2, tmp3, tmp4)
-      tmp1.rewind
-      tmp1.truncate(0)
-      tmp2.rewind
-      tmp2.truncate(0)
-      tmp1, tmp2, tmp3, tmp4 = tmp3, tmp4, tmp1, tmp2
-    end
-      #dump_objsfile(:tmp1, tmp1)
-      #dump_objsfile(:tmp2, tmp2)
-      #dump_objsfile(:tmp3, tmp3)
-      #dump_objsfile(:tmp4, tmp4)
-    extsort_by_strip_cv(tmp1, y)
-  ensure
-    tmp1.close(true) if tmp1
-    tmp2.close(true) if tmp2
-    tmp3.close(true) if tmp3
-    tmp4.close(true) if tmp4
+  def to_tb
+    tb = Tb.new
+    self.each {|pairs|
+      pairs.each {|k, v|
+        unless tb.has_field? k
+          tb.define_field(k)
+        end
+      }
+      tb.insert pairs
+    }
+    tb
   end
-  private :extsort_by_internal
 
-  def extsort_by_first_split(tmp1, tmp2, cmpvalue_from, memsize)
-    prevobj_cv = nil
-    tmp_current, tmp_another = tmp1, tmp2
-    buf = []
-    buf_size = 0
-    buf_mode = true
-    self.each_with_index {|obj, i|
-      obj_cv = cmpvalue_from.call(obj)
-      #p [obj, obj_cv]
-      #p [prevobj_cv, buf_mode, obj, obj_cv]
-      if buf_mode
-        dumped = Marshal.dump([obj_cv, obj])
-        buf << [obj_cv, i, dumped]
-        buf_size += dumped.size
-        if memsize < buf_size
-          buf.sort!
-          buf.each {|_, _, d|
-            tmp_current.write d
-          }
-          prevobj_cv, = buf.last
-          buf.clear
-          buf_mode = false
+  def write_to_csv(io, with_header=true)
+    stream = nil
+    header = []
+    fgen = fnew = nil
+    self.with_cumulative_header {|header0|
+      if !with_header
+        stream = true
+      elsif header0
+        stream = true
+        io.puts Tb.csv_encode_row(header0)
+      else
+        stream = false
+        fgen, fnew = Tb::FileEnumerator.gen_new
+      end
+    }.each {|pairs, header1|
+      pairs = Hash[pairs] unless pairs.respond_to? :has_key?
+      header = header1
+      if stream
+        fs = header.dup
+        while !fs.empty? && !pairs.has_key?(fs.last)
+          fs.pop
         end
-      elsif prevobj_cv <= obj_cv
-        Marshal.dump([obj_cv, obj], tmp_current)
-        prevobj_cv = obj_cv
+        ary = fs.map {|f| pairs[f] }
+        io.puts Tb.csv_encode_row(ary)
       else
-        dumped = Marshal.dump([obj_cv, obj])
-        Marshal.dump(nil, tmp_current)
-        buf = [[obj_cv, i, dumped]]
-        buf_size = dumped.size
-        buf_mode = true
-        tmp_current, tmp_another = tmp_another, tmp_current
+        fgen.call Hash[pairs]
       end
     }
-    if buf_mode
-      buf.sort!
-      buf.each {|_, _, d|
-        tmp_current.write d
-      }
-    end
-    if !buf_mode || !buf.empty?
-      Marshal.dump(nil, tmp_current)
-    end
-  end
-  private :extsort_by_first_split
-
-  def extsort_by_merge(src1, src2, dst1, dst2)
-    src1.rewind
-    src2.rewind
-    obj1_cv, obj1 = obj1_pair = Marshal.load(src1)
-    obj2_cv, obj2 = obj2_pair = Marshal.load(src2)
-    prefer1 = true
-    while true
-      cmp = obj1_cv <=> obj2_cv
-      if prefer1 ? cmp > 0 : cmp >= 0
-        obj1_pair, obj1_cv, obj1, src1, obj2_pair, obj2_cv, obj2, src2 = obj2_pair, obj2_cv, obj2, src2, obj1_pair, obj1_cv, obj1, src1
-        prefer1 = !prefer1
-      end
-      Marshal.dump([obj1_cv, obj1], dst1)
-      obj1_cv, obj1 = obj1_pair = Marshal.load(src1)
-      if !obj1_pair
-        begin
-          Marshal.dump(obj2_pair, dst1)
-          obj2_pair = Marshal.load(src2)
-        end until !obj2_pair
-        Marshal.dump(nil, dst1)
-        dst1, dst2 = dst2, dst1
-        break if src1.eof?
-        break if src2.eof?
-        obj1_cv, obj1 = obj1_pair = Marshal.load(src1)
-        obj2_cv, obj2 = obj2_pair = Marshal.load(src2)
+    if !stream
+      if with_header
+        io.puts Tb.csv_encode_row(header)
       end
-    end
-    if !src1.eof?
-      restsrc = src1
-    elsif !src2.eof?
-      restsrc = src2
-    else
-      return
-    end
-    until restsrc.eof?
-      restobj_pair = Marshal.load(restsrc)
-      Marshal.dump(restobj_pair, dst1)
-    end
-  end
-  private :extsort_by_merge
-
-  def extsort_by_strip_cv(tmp1, y)
-    tmp1.rewind
-    while true
-      pair = Marshal.load(tmp1)
-      break if !pair
-      _, obj = pair
-      y.yield obj
+      fnew.call.each {|pairs|
+        fs = header.dup
+        while !fs.empty? && !pairs.has_key?(fs.last)
+          fs.pop
+        end
+        ary = fs.map {|f| pairs[f] }
+        io.puts Tb.csv_encode_row(ary)
+      }
     end
   end
-  private :extsort_by_strip_cv
 
-  # splits self by _boundary_p_ which is called with adjacent two elements.
-  #
-  # _before_group_ is called before each group with the first element.
-  # _after_group_ is called after each group with the last element.
-  # _body_ is called for each element.
-  #
-  def each_group_element(boundary_p, before_group, body, after_group)
-    prev = nil
-    first = true
-    self.each {|curr|
-      if first
-        before_group.call(curr)
-        body.call(curr)
-        prev = curr
-        first = false
-      elsif boundary_p.call(prev, curr)
-        after_group.call(prev)
-        before_group.call(curr)
-        body.call(curr)
-        prev = curr
-      else
-        body.call(curr)
-        prev = curr
-      end
+  def write_to_json(out)
+    require 'json'
+    out.print "["
+    sep = nil
+    self.each {|pairs|
+      out.print sep if sep
+      out.print JSON.pretty_generate(Hash[pairs.to_a])
+      sep = ",\n"
     }
-    if !first
-      after_group.call(prev)
-    end
+    out.puts "]"
+    nil
   end
 
-  def lazy_map
-    Enumerator.new {|y|
-      self.each {|*vs|
-        y.yield(yield(*vs))
+  def extsort_by(opts={}, &cmpvalue_from)
+    Tb::Enumerator.new {|ty|
+      header = []
+      er = Enumerator.new {|y|
+        self.with_cumulative_header {|header0|
+          header = header0 if header0
+        }.each {|pairs, header1|
+          header = header1
+          y.yield pairs
+        }
+        ty.set_header header
+      }
+      er.extsort_by(opts, &cmpvalue_from).each {|pairs|
+        ty.yield pairs
       }
     }
   end