tb 0.2 → 0.3

data/lib/tb/enumerable.rb

@@ -1,6 +1,6 @@
 # lib/tb/enumerable.rb - extensions for Enumerable
 #
-# Copyright (C) 2010 Tanaka Akira  <akr@fsij.org>
+# Copyright (C) 2010-2012 Tanaka Akira  <akr@fsij.org>
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are met:
@@ -117,13 +117,13 @@ module Enumerable
     end
     value_selector = tb_cat_selector_proc(args.pop)
     key_selectors = args.map {|a| tb_cat_selector_proc(a) }
-    has_seed = opts.include? :seed
+    has_seed = opts.has_key? :seed
     seed_value = opts[:seed]
-    if opts.include?(:update) && opts.include?(:op)
+    if opts.has_key?(:update) && opts.has_key?(:op)
       raise ArgumentError, "both :op and :update option specified"
-    elsif opts.include? :update
+    elsif opts.has_key? :update
       update_proc = opts[:update].to_proc
-    elsif opts.include? :op
+    elsif opts.has_key? :op
       op_proc = opts[:op].to_proc
       update_proc = lambda {|ks, s, v| op_proc.call(s, v) }
     else
@@ -139,11 +139,11 @@ module Enumerable
       h = result
       0.upto(ks.length-2) {|i|
         k = ks[i]
-        h[k] = {} if !h.include?(k)
+        h[k] = {} if !h.has_key?(k)
         h = h[k]
       }
       lastk = ks.last
-      if !h.include?(lastk)
+      if !h.has_key?(lastk)
         if has_seed
           h[lastk] = update_proc.call(ks, seed_value, v)
         else
@@ -281,4 +281,195 @@ module Enumerable
     tb_categorize(*(args + [lambda {|e| 1 }, {:update => lambda {|ks, s, v| s + v }}]))
   end
 
+  def dump_objsfile(title, tempfile)
+    tempfile.flush
+    path = tempfile
+    a = []
+    open(path) {|f|
+      until f.eof?
+        pair = Marshal.load(f)
+        a << (pair ? pair.last : :sep)
+      end
+    }
+    puts "#{title}: #{a.inspect}"
+  end
+  private :dump_objsfile
+
+  def extsort_by(opts={}, &cmpvalue_from)
+    memsize = opts[:memsize] || 10000000
+    Enumerator.new {|y|
+      extsort_by_internal(memsize, cmpvalue_from, y)
+    }
+  end
+
+  def extsort_by_internal(memsize, cmpvalue_from, y)
+    tmp1 = Tempfile.new("tbsortA")
+    tmp2 = Tempfile.new("tbsortB")
+    extsort_by_first_split(tmp1, tmp2, cmpvalue_from, memsize)
+    if tmp1.size == 0 && tmp2.size == 0
+      return Enumerator.new {|_| }
+    end
+    tmp3 = Tempfile.new("tbsortC")
+    tmp4 = Tempfile.new("tbsortD")
+    while tmp2.size != 0
+      #dump_objsfile(:tmp1, tmp1)
+      #dump_objsfile(:tmp2, tmp2)
+      #dump_objsfile(:tmp3, tmp3)
+      #dump_objsfile(:tmp4, tmp4)
+      extsort_by_merge(tmp1, tmp2, tmp3, tmp4)
+      tmp1.rewind
+      tmp1.truncate(0)
+      tmp2.rewind
+      tmp2.truncate(0)
+      tmp1, tmp2, tmp3, tmp4 = tmp3, tmp4, tmp1, tmp2
+    end
+      #dump_objsfile(:tmp1, tmp1)
+      #dump_objsfile(:tmp2, tmp2)
+      #dump_objsfile(:tmp3, tmp3)
+      #dump_objsfile(:tmp4, tmp4)
+    extsort_by_strip_cv(tmp1, y)
+  ensure
+    tmp1.close(true) if tmp1
+    tmp2.close(true) if tmp2
+    tmp3.close(true) if tmp3
+    tmp4.close(true) if tmp4
+  end
+  private :extsort_by_internal
+
+  def extsort_by_first_split(tmp1, tmp2, cmpvalue_from, memsize)
+    prevobj_cv = nil
+    tmp_current, tmp_another = tmp1, tmp2
+    buf = []
+    buf_size = 0
+    buf_mode = true
+    self.each_with_index {|obj, i|
+      obj_cv = cmpvalue_from.call(obj)
+      #p [obj, obj_cv]
+      #p [prevobj_cv, buf_mode, obj, obj_cv]
+      if buf_mode
+        dumped = Marshal.dump([obj_cv, obj])
+        buf << [obj_cv, i, dumped]
+        buf_size += dumped.size
+        if memsize < buf_size
+          buf.sort!
+          buf.each {|_, _, d|
+            tmp_current.write d
+          }
+          prevobj_cv, = buf.last
+          buf.clear
+          buf_mode = false
+        end
+      elsif prevobj_cv <= obj_cv
+        Marshal.dump([obj_cv, obj], tmp_current)
+        prevobj_cv = obj_cv
+      else
+        dumped = Marshal.dump([obj_cv, obj])
+        Marshal.dump(nil, tmp_current)
+        buf = [[obj_cv, i, dumped]]
+        buf_size = dumped.size
+        buf_mode = true
+        tmp_current, tmp_another = tmp_another, tmp_current
+      end
+    }
+    if buf_mode
+      buf.sort!
+      buf.each {|_, _, d|
+        tmp_current.write d
+      }
+    end
+    if !buf_mode || !buf.empty?
+      Marshal.dump(nil, tmp_current)
+    end
+  end
+  private :extsort_by_first_split
+
+  def extsort_by_merge(src1, src2, dst1, dst2)
+    src1.rewind
+    src2.rewind
+    obj1_cv, obj1 = obj1_pair = Marshal.load(src1)
+    obj2_cv, obj2 = obj2_pair = Marshal.load(src2)
+    prefer1 = true
+    while true
+      cmp = obj1_cv <=> obj2_cv
+      if prefer1 ? cmp > 0 : cmp >= 0
+        obj1_pair, obj1_cv, obj1, src1, obj2_pair, obj2_cv, obj2, src2 = obj2_pair, obj2_cv, obj2, src2, obj1_pair, obj1_cv, obj1, src1
+        prefer1 = !prefer1
+      end
+      Marshal.dump([obj1_cv, obj1], dst1)
+      obj1_cv, obj1 = obj1_pair = Marshal.load(src1)
+      if !obj1_pair
+        begin
+          Marshal.dump(obj2_pair, dst1)
+          obj2_pair = Marshal.load(src2)
+        end until !obj2_pair
+        Marshal.dump(nil, dst1)
+        dst1, dst2 = dst2, dst1
+        break if src1.eof?
+        break if src2.eof?
+        obj1_cv, obj1 = obj1_pair = Marshal.load(src1)
+        obj2_cv, obj2 = obj2_pair = Marshal.load(src2)
+      end
+    end
+    if !src1.eof?
+      restsrc = src1
+    elsif !src2.eof?
+      restsrc = src2
+    else
+      return
+    end
+    until restsrc.eof?
+      restobj_pair = Marshal.load(restsrc)
+      Marshal.dump(restobj_pair, dst1)
+    end
+  end
+  private :extsort_by_merge
+
+  def extsort_by_strip_cv(tmp1, y)
+    tmp1.rewind
+    while true
+      pair = Marshal.load(tmp1)
+      break if !pair
+      _, obj = pair
+      y.yield obj
+    end
+  end
+  private :extsort_by_strip_cv
+
+  # splits self by _boundary_p_ which is called with adjacent two elements.
+  #
+  # _before_group_ is called before each group with the first element.
+  # _after_group_ is called after each group with the last element.
+  # _body_ is called for each element.
+  #
+  def each_group_element(boundary_p, before_group, body, after_group)
+    prev = nil
+    first = true
+    self.each {|curr|
+      if first
+        before_group.call(curr)
+        body.call(curr)
+        prev = curr
+        first = false
+      elsif boundary_p.call(prev, curr)
+        after_group.call(prev)
+        before_group.call(curr)
+        body.call(curr)
+        prev = curr
+      else
+        body.call(curr)
+        prev = curr
+      end
+    }
+    if !first
+      after_group.call(prev)
+    end
+  end
+
+  def lazy_map
+    Enumerator.new {|y|
+      self.each {|*vs|
+        y.yield(yield(*vs))
+      }
+    }
+  end
 end

data/lib/tb/enumerator.rb

@@ -0,0 +1,73 @@
+# Copyright (C) 2012 Tanaka Akira  <akr@fsij.org>
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 
+#  1. Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#  2. Redistributions in binary form must reproduce the above
+#     copyright notice, this list of conditions and the following
+#     disclaimer in the documentation and/or other materials provided
+#     with the distribution.
+#  3. The name of the author may not be used to endorse or promote
+#     products derived from this software without specific prior
+#     written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+# GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+class Tb::Yielder
+  def initialize(header_proc, each_proc)
+    @header_proc_called = false
+    @header_proc = header_proc
+    @each_proc = each_proc
+  end
+  attr_reader :header_proc_called
+
+  def set_header(header)
+    raise ArgumentError, "set_header called twice" if @header_proc_called
+    @header_proc_called = true
+    @header_proc.call(header) if @header_proc
+  end
+
+  def yield(*args)
+    if !@header_proc_called
+      set_header(nil)
+    end
+    @each_proc.call(*args)
+  end
+  alias << yield
+end
+
+class Tb::Enumerator
+  include Tb::Enum
+
+  def initialize(&enumerator_proc)
+    @enumerator_proc = enumerator_proc
+  end
+
+  def each(&each_proc)
+    yielder = Tb::Yielder.new(nil, each_proc)
+    @enumerator_proc.call(yielder)
+    nil
+  end
+
+  def header_and_each(header_proc, &each_proc)
+    yielder = Tb::Yielder.new(header_proc, each_proc)
+    @enumerator_proc.call(yielder)
+    if !yielder.header_proc_called
+      header_proc.call(nil)
+    end
+    nil
+  end
+end

data/lib/tb/fieldset.rb

@@ -1,30 +1,38 @@
 # lib/tb/fieldset.rb - Tb::FieldSet class
 #
-# Copyright (C) 2011 Tanaka Akira  <akr@fsij.org>
+# Copyright (C) 2011-2012 Tanaka Akira  <akr@fsij.org>
 # 
 # Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
+# modification, are permitted provided that the following conditions
+# are met:
 # 
-#  1. Redistributions of source code must retain the above copyright notice, this
-#     list of conditions and the following disclaimer.
-#  2. Redistributions in binary form must reproduce the above copyright notice,
-#     this list of conditions and the following disclaimer in the documentation
-#     and/or other materials provided with the distribution.
-#  3. The name of the author may not be used to endorse or promote products
-#     derived from this software without specific prior written permission.
+#  1. Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#  2. Redistributions in binary form must reproduce the above
+#     copyright notice, this list of conditions and the following
+#     disclaimer in the documentation and/or other materials provided
+#     with the distribution.
+#  3. The name of the author may not be used to endorse or promote
+#     products derived from this software without specific prior
+#     written permission.
 # 
-# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
-# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
-# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
-# EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
-# OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
-# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
-# OF SUCH DAMAGE.
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+# GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 class Tb::FieldSet
+  def self.normalize(header)
+    Tb::FieldSet.new(*header).header
+  end
+
   def initialize(*fs)
     @header = []
     @field2index = {}

data/lib/tb/fileenumerator.rb

@@ -0,0 +1,365 @@
+# Copyright (C) 2012 Tanaka Akira  <akr@fsij.org>
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 
+#  1. Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#  2. Redistributions in binary form must reproduce the above
+#     copyright notice, this list of conditions and the following
+#     disclaimer in the documentation and/or other materials provided
+#     with the distribution.
+#  3. The name of the author may not be used to endorse or promote
+#     products derived from this software without specific prior
+#     written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+# GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+module Enumerable
+  # creates a Tb::FileEnumerator object.
+  #
+  def to_fileenumerator
+    Tb::FileEnumerator.new_tempfile {|gen|
+      self.each {|*objs|
+        gen.call(*objs)
+      }
+    }
+  end
+end
+
+# Tb::FileEnumerator is an enumerator backed by a temporally file.
+#
+# An instance of Tb::FileEnumerator can be used just once,
+# except if Tb::FileEnumerator#use is explicitly used.
+#
+# After the use, the temporally file is removed.
+# If the object is not used, the temporally file is removed by GC
+# as usual Tempfile object.
+#
+class Tb::FileEnumerator
+  include Enumerable
+
+  class Builder
+    def initialize(klass)
+      @klass = klass
+      @tempfile = Tempfile.new("tb")
+      @tempfile.binmode
+    end
+
+    def gen(*objs)
+      Marshal.dump(objs, @tempfile)
+    end
+
+    def new
+      @tempfile.close
+      @klass.new(
+        lambda { open(@tempfile.path, "rb") },
+        lambda { @tempfile.close(true) })
+    end
+  end
+
+  def self.builder
+    Builder.new(Tb::FileEnumerator)
+  end
+
+  def self.new_tempfile
+    gen, new = self.gen_new
+    yield gen
+    new.call
+  end
+
+  def self.gen_new
+    builder = self.builder
+    return builder.method(:gen), builder.method(:new)
+  end
+
+  def initialize(open_func, remove_func)
+    @use_count = 0
+    @open_func = open_func
+    @remove_func = remove_func
+  end
+
+  def use_count_up
+    @use_count += 1
+  end
+  private :use_count_up
+
+  def use_count_down
+    @use_count -= 1
+    if @use_count == 0
+      @remove_func.call
+      @open_func = @remove_func = nil
+    end
+  end
+  private :use_count_down
+
+  # delay removing the tempfile until the given block is finished.
+  def use
+    if !@open_func
+      raise ArgumentError, "FileEnumerator reused."
+    end
+    use_count_up
+    begin
+      yield
+    ensure
+      use_count_down
+    end
+  end
+
+  def each
+    if block_given?
+      self.use {
+        begin
+          io = @open_func.call
+          while true
+            objs = Marshal.load(io)
+            yield(*objs)
+          end
+        rescue EOFError
+        ensure
+          io.close 
+        end
+      }
+    else
+      Reader.new(self)
+    end
+  end
+
+  def open_reader
+    reader = Reader.new(self)
+    reader.use {
+      yield reader
+    }
+  end
+
+  class Reader
+    def initialize(fileenumerator)
+      @use_count = 0
+      @fileenumerator = fileenumerator
+      @fileenumerator.send(:use_count_up)
+      @io = @fileenumerator.instance_eval { @open_func.call }
+      peek_reset
+    end
+
+    def closed?
+      @io.nil?
+    end
+    
+    def finalize
+      @io.close
+      @io = nil
+      @fileenumerator.send(:use_count_down)
+      peek_reset
+    end
+    private :finalize
+
+    def use_begin
+      @use_count += 1
+    end
+
+    def use_end
+      @use_count -= 1
+      if @use_count == 0
+        finalize
+      end
+    end
+
+    def use
+      use_begin
+      begin
+        yield
+      ensure
+        use_end
+      end
+    end
+
+    def peek_reset
+      @peeked = false
+      @peeked_objs = nil
+      if @io
+        @pos = @io.pos
+      else
+        @pos = nil
+      end
+    end
+    private :peek_reset
+
+    def pos
+      @pos
+    end
+
+    def pos=(val)
+      @io.seek(val)
+      peek_reset
+      nil
+    end
+
+    def peek_values
+      if !@io
+        raise StopIteration
+      end
+      if !@peeked
+        begin
+          objs = Marshal.load(@io)
+        rescue EOFError
+          if @use_count == 0
+            finalize
+          end
+          raise StopIteration
+        end
+        @peeked = true
+        @peeked_objs = objs
+      end
+      @peeked_objs
+    end
+
+    def peek
+      result = self.peek_values
+      if result.kind_of?(Array) && result.length == 1
+        result = result[0]
+      end
+      result
+    end
+
+    def next_values
+      result = self.peek_values
+      peek_reset
+      result
+    end
+
+    def next
+      result = self.peek
+      peek_reset
+      result
+    end
+
+    def rewind
+      if !@io
+        raise ArgumentError, "already closed."
+      end
+      @io.rewind
+      peek_reset
+      nil
+    end
+
+    def subeach_by(&distinguish_value)
+      Enumerator.new {|y|
+        begin
+          vs = self.peek_values
+        rescue StopIteration
+          next
+        end
+        dv = distinguish_value.call(*vs)
+        while true
+          y.yield(*vs)
+          self.next_values
+          begin
+            next_vs = self.peek_values
+          rescue StopIteration
+            break
+          end
+          next_dv = distinguish_value.call(*next_vs)
+          if dv != next_dv
+            break
+          end
+          vs = next_vs
+          dv = next_dv
+        end
+        nil
+      }
+    end
+  end
+end
+
+module Tb::Enum
+  # creates a Tb::FileHeaderEnumerator object.
+  #
+  def to_fileenumerator
+    hbuilder = Tb::FileHeaderEnumerator.builder
+    self.with_header {|header|
+      if header
+        hbuilder.header.concat(header - hbuilder.header)
+      end
+    }.each {|pairs|
+      hbuilder.gen(pairs)
+    }
+    hbuilder.new
+  end
+end
+
+class Tb::FileHeaderEnumerator < Tb::FileEnumerator
+  include Tb::Enum
+
+  class HBuilder
+    def initialize(klass)
+      @klass = klass
+      @tempfile = Tempfile.new("tb")
+      @tempfile.binmode
+      @header = []
+    end
+    attr_reader :header
+
+    def gen(*objs)
+      @header |= objs[0].keys
+      Marshal.dump(objs, @tempfile)
+    end
+
+    def new
+      @tempfile.close
+      @klass.new(
+        @header,
+        lambda { open(@tempfile.path, "rb") },
+        lambda { @tempfile.close(true) })
+    end
+  end
+
+  def self.builder
+    HBuilder.new(Tb::FileHeaderEnumerator)
+  end
+
+  def self.gen_new
+    hgen = self.builder
+    return hgen.method(:gen), hgen.method(:new)
+  end
+
+  def initialize(header, open_func, remove_func)
+    super open_func, remove_func
+    @header = header
+  end
+
+  def header_and_each(header_proc)
+    self.use {
+      header_proc.call(@header) if header_proc
+      begin
+        io = @open_func.call
+        while true
+          objs = Marshal.load(io)
+          yield(*objs)
+        end
+      rescue EOFError
+      ensure
+        io.close 
+      end
+    }
+  end
+
+  def each(&block)
+    if block_given?
+      header_and_each(nil, &block)
+    else
+      Tb::FileEnumerator::Reader.new(self)
+    end
+  end
+
+end