tb 0.2 → 0.3

data/lib/tb/catreader.rb

@@ -1,131 +1,48 @@
 # lib/tb/catreader.rb - Tb::CatReader class
 #
-# Copyright (C) 2011 Tanaka Akira  <akr@fsij.org>
+# Copyright (C) 2011-2012 Tanaka Akira  <akr@fsij.org>
 # 
 # Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
+# modification, are permitted provided that the following conditions
+# are met:
 # 
-#  1. Redistributions of source code must retain the above copyright notice, this
-#     list of conditions and the following disclaimer.
-#  2. Redistributions in binary form must reproduce the above copyright notice,
-#     this list of conditions and the following disclaimer in the documentation
-#     and/or other materials provided with the distribution.
-#  3. The name of the author may not be used to endorse or promote products
-#     derived from this software without specific prior written permission.
+#  1. Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#  2. Redistributions in binary form must reproduce the above
+#     copyright notice, this list of conditions and the following
+#     disclaimer in the documentation and/or other materials provided
+#     with the distribution.
+#  3. The name of the author may not be used to endorse or promote
+#     products derived from this software without specific prior
+#     written permission.
 # 
-# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
-# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
-# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
-# EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
-# OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
-# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
-# OF SUCH DAMAGE.
-
-class Tb::CatReader
-  def self.open(filenames, numeric=false)
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+# GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+module Tb::CatReader
+  def self.open(filenames, numeric=false, with_filename=false)
     readers = []
     filenames.each {|f|
-      readers << Tb::Reader.open(f, numeric ? {:numeric=>true} : {})
+      r = Tb.open_reader(f, numeric ? {:numeric=>true} : {})
+      if with_filename
+        r = r.newfield("filename") { f }
+      end
+      readers << r
     }
-    r = Tb::CatReader.new(readers, readers, numeric)
+    r = readers.first.cat(*readers[1..-1])
     if block_given?
-      begin
-        yield r
-      ensure
-        r.close
-      end
+      yield r
     else
       r
     end
   end
-
-  def initialize(readers, also_close, numeric)
-    @readers = readers.dup
-    @also_close = also_close
-    @numeric = numeric
-    @fieldset = nil
-  end
-
-  def header
-    return @fieldset.header if @fieldset
-    if @numeric
-      @fieldset = Tb::FieldSet.new
-    else
-      h = {}
-      @readers.each {|r|
-        r.header.each {|f|
-          if !h[f]
-            h[f] = h.size
-          end
-        }
-      }
-      @fieldset = Tb::FieldSet.new(*h.keys.sort_by {|f| h[f] })
-    end
-    return @fieldset.header
-  end
-
-  def index_from_field_ex(f)
-    self.header
-    @fieldset.index_from_field_ex(f)
-  end
-
-  def index_from_field(f)
-    self.header
-    @fieldset.index_from_field(f)
-  end
-
-  def field_from_index_ex(i)
-    self.header
-    @fieldset.field_from_index_ex(i)
-  end
-
-  def field_from_index(i)
-    self.header
-    @fieldset.field_from_index(i)
-  end
-
-  def shift
-    self.header
-    while !@readers.empty?
-      r = @readers.first
-      ary = r.shift
-      if ary
-        h = r.header
-        ary2 = []
-        ary.each_with_index {|v,i|
-          f = h[i]
-          i2 = @fieldset.index_from_field_ex(f)
-          ary2[i2] = v
-        }
-        return ary2
-      else
-        @readers.shift
-      end
-    end
-    nil
-  end
-
-  def each
-    while ary = self.shift
-      yield ary
-    end
-    nil
-  end
-
-  def read_all
-    result = []
-    while ary = self.shift
-      result << ary
-    end
-    result
-  end
-
-  def close
-    if @also_close
-      @also_close.each {|x| x.close }
-    end
-  end
 end

data/lib/tb/cmd_cat.rb

@@ -1,33 +1,41 @@
-# Copyright (C) 2011 Tanaka Akira  <akr@fsij.org>
+# Copyright (C) 2011-2012 Tanaka Akira  <akr@fsij.org>
 # 
 # Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
+# modification, are permitted provided that the following conditions
+# are met:
 # 
-#  1. Redistributions of source code must retain the above copyright notice, this
-#     list of conditions and the following disclaimer.
-#  2. Redistributions in binary form must reproduce the above copyright notice,
-#     this list of conditions and the following disclaimer in the documentation
-#     and/or other materials provided with the distribution.
-#  3. The name of the author may not be used to endorse or promote products
-#     derived from this software without specific prior written permission.
+#  1. Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#  2. Redistributions in binary form must reproduce the above
+#     copyright notice, this list of conditions and the following
+#     disclaimer in the documentation and/or other materials provided
+#     with the distribution.
+#  3. The name of the author may not be used to endorse or promote
+#     products derived from this software without specific prior
+#     written permission.
 # 
-# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
-# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
-# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
-# EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
-# OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
-# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
-# OF SUCH DAMAGE.
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+# GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 Tb::Cmd.subcommands << 'cat'
 
+Tb::Cmd.default_option[:opt_cat_with_filename] = nil
+
 def (Tb::Cmd).op_cat
   op = OptionParser.new
-  op.banner = 'Usage: tb cat [OPTS] [TABLE ...]'
+  op.banner = "Usage: tb cat [OPTS] [TABLE ...]\n" +
+    "Concatenate tables vertically."
   define_common_option(op, "hNo", "--no-pager")
+  op.def_option('-H', '--with-filename', 'add filename column') { Tb::Cmd.opt_cat_with_filename = true }
   op
 end
 
@@ -54,12 +62,8 @@ def (Tb::Cmd).main_cat(argv)
   op_cat.parse!(argv)
   exit_if_help('cat')
   argv = ['-'] if argv.empty?
-  creader = Tb::CatReader.open(argv, Tb::Cmd.opt_N)
-  header = creader.header
-  with_table_stream_output {|gen|
-    gen << header if !Tb::Cmd.opt_N
-    creader.each {|ary|
-      gen << ary
-    }
+  creader = Tb::CatReader.open(argv, Tb::Cmd.opt_N, Tb::Cmd.opt_cat_with_filename)
+  with_output {|out|
+    creader.write_to_csv(out, !Tb::Cmd.opt_N)
   }
 end

data/lib/tb/cmd_consecutive.rb

@@ -1,26 +1,30 @@
-# Copyright (C) 2011 Tanaka Akira  <akr@fsij.org>
+# Copyright (C) 2011-2012 Tanaka Akira  <akr@fsij.org>
 # 
 # Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
+# modification, are permitted provided that the following conditions
+# are met:
 # 
-#  1. Redistributions of source code must retain the above copyright notice, this
-#     list of conditions and the following disclaimer.
-#  2. Redistributions in binary form must reproduce the above copyright notice,
-#     this list of conditions and the following disclaimer in the documentation
-#     and/or other materials provided with the distribution.
-#  3. The name of the author may not be used to endorse or promote products
-#     derived from this software without specific prior written permission.
+#  1. Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#  2. Redistributions in binary form must reproduce the above
+#     copyright notice, this list of conditions and the following
+#     disclaimer in the documentation and/or other materials provided
+#     with the distribution.
+#  3. The name of the author may not be used to endorse or promote
+#     products derived from this software without specific prior
+#     written permission.
 # 
-# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
-# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
-# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
-# EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
-# OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
-# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
-# OF SUCH DAMAGE.
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+# GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 Tb::Cmd.subcommands << 'consecutive'
 
@@ -28,7 +32,8 @@ Tb::Cmd.default_option[:opt_consecutive_n] = 2
 
 def (Tb::Cmd).op_consecutive
   op = OptionParser.new
-  op.banner = 'Usage: tb consecutive [OPTS] [TABLE ...]'
+  op.banner = "Usage: tb consecutive [OPTS] [TABLE ...]\n" +
+    "Concatenate consecutive rows."
   define_common_option(op, "hNo", "--no-pager")
   op.def_option('-n NUM', 'gather NUM records.  (default: 2)') {|n| Tb::Cmd.opt_consecutive_n = n.to_i }
   op
@@ -53,27 +58,32 @@ def (Tb::Cmd).main_consecutive(argv)
   exit_if_help('consecutive')
   argv = ['-'] if argv.empty?
   creader = Tb::CatReader.open(argv, Tb::Cmd.opt_N)
-  consecutive_header = []
-  creader.header.each {|f|
-    Tb::Cmd.opt_consecutive_n.times {|i|
-      consecutive_header << "#{f}_#{i+1}"
-    }
-  }
-  with_table_stream_output {|gen|
-    gen.output_header consecutive_header
+  er = Tb::Enumerator.new {|y|
     buf = []
-    creader.each {|ary|
-      buf << ary
+    empty = true
+    creader.with_cumulative_header {|header0|
+      if header0
+        y.set_header header0.map {|f| (1..Tb::Cmd.opt_consecutive_n).map {|i| "#{f}_#{i}" } }.flatten(1)
+      end
+    }.each {|pairs, header|
+      buf << pairs
       if buf.length == Tb::Cmd.opt_consecutive_n
-        ary2 = []
-        buf.each_with_index {|a, i|
-          a.each_with_index {|e, j|
-            ary2[j*Tb::Cmd.opt_consecutive_n + i] = e
+        pairs2 = []
+        header.each {|f|
+          Tb::Cmd.opt_consecutive_n.times {|i|
+            ps = buf[i]
+            next if !ps.has_key?(f)
+            v = ps[f]
+            pairs2 << ["#{f}_#{i+1}", v]
           }
         }
-        gen << ary2
+        empty = false
+        y.yield Tb::Pairs.new(pairs2)
         buf.shift
       end
     }
   }
+  with_output {|out|
+    er.write_to_csv(out, !Tb::Cmd.opt_N)
+  }
 end

data/lib/tb/cmd_crop.rb

@@ -1,26 +1,30 @@
-# Copyright (C) 2011 Tanaka Akira  <akr@fsij.org>
+# Copyright (C) 2011-2012 Tanaka Akira  <akr@fsij.org>
 # 
 # Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
+# modification, are permitted provided that the following conditions
+# are met:
 # 
-#  1. Redistributions of source code must retain the above copyright notice, this
-#     list of conditions and the following disclaimer.
-#  2. Redistributions in binary form must reproduce the above copyright notice,
-#     this list of conditions and the following disclaimer in the documentation
-#     and/or other materials provided with the distribution.
-#  3. The name of the author may not be used to endorse or promote products
-#     derived from this software without specific prior written permission.
+#  1. Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#  2. Redistributions in binary form must reproduce the above
+#     copyright notice, this list of conditions and the following
+#     disclaimer in the documentation and/or other materials provided
+#     with the distribution.
+#  3. The name of the author may not be used to endorse or promote
+#     products derived from this software without specific prior
+#     written permission.
 # 
-# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
-# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
-# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
-# EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
-# OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
-# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
-# OF SUCH DAMAGE.
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+# GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 Tb::Cmd.subcommands << 'crop'
 
@@ -28,7 +32,8 @@ Tb::Cmd.default_option[:opt_crop_range] = nil
 
 def (Tb::Cmd).op_crop
   op = OptionParser.new
-  op.banner = 'Usage: tb crop [OPTS] [TABLE ...]'
+  op.banner = "Usage: tb crop [OPTS] [TABLE ...]\n" +
+    "Extract rectangle in a table."
   define_common_option(op, "ho", "--no-pager")
   op.def_option('-r RANGE', 'range.  i.e. "2,1-4,3", "B1:D3"') {|arg| Tb::Cmd.opt_crop_range = arg }
   op
@@ -62,43 +67,72 @@ def (Tb::Cmd).main_crop(argv)
     end
   end
   if stream
-    with_table_stream_output {|gen|
-      Tb::CatReader.open(argv, true) {|tblreader|
-        rownum = 1
-        tblreader.each {|ary|
-          if range_row2 < rownum
-            break
-          end
-          if range_row1 <= rownum
-            if range_col2 < ary.length
-              ary[range_col2..-1] = []
-            end
-            if 1 < range_col1
-              ary[0...(range_col1-1)] = []
-            end
-            gen << ary
-          end
-          rownum += 1
-        }
+    creader = Tb::CatReader.open(argv, true)
+    er = Tb::Enumerator.new {|y|
+      rownum = 1
+      creader.each {|pairs|
+        if range_row2 < rownum
+          break
+        end
+        if range_row1 <= rownum
+          pairs2 = pairs.reject {|f, v|
+            f = f.to_i
+            f < range_col1 || range_col2 < f
+          }
+          y.yield pairs2
+        end
+        rownum += 1
       }
     }
+    with_output {|out|
+      er.write_to_csv(out, false)
+    }
   else
-    arys = []
-    Tb::CatReader.open(argv, true) {|tblreader|
-      tblreader.each {|a|
-        a.pop while !a.empty? && (a.last.nil? || a.last == '')
-        arys << a
+    creader = Tb::CatReader.open(argv, true)
+    last_nonempty_row = nil
+    lmargin_min = nil
+    ter = Enumerator.new {|y|
+      numrows = 0
+      creader.each {|pairs|
+        ary = []
+        pairs.each {|f, v|
+          ary[f.to_i-1] = v
+        }
+        while !ary.empty? && (ary.last.nil? || ary.last == '')
+          ary.pop
+        end
+        if numrows == 0 && ary.empty?
+          next
+        end
+        if !ary.empty?
+          lmargin = 0
+          while lmargin < ary.length
+            if !ary[lmargin].nil? && ary[lmargin] != ''
+              break 
+            end
+            lmargin += 1
+          end
+          if lmargin_min.nil? || lmargin < lmargin_min
+            lmargin_min = lmargin
+          end
+        end
+        last_nonempty_row = numrows if !ary.empty?
+        y.yield ary
+        numrows += 1
+      }
+    }.to_fileenumerator
+    er = Tb::Enumerator.new {|y|
+      ter.each_with_index {|ary, rownum|
+        if last_nonempty_row < rownum
+          break
+        end
+        ary.slice!(0, lmargin_min)
+        pairs = Tb::Pairs.new(ary.map.with_index {|v, i| ["#{i+1}", v]})
+        y.yield pairs
       }
     }
-    arys.pop while !arys.empty? && arys.last.all? {|v| v.nil? || v == '' }
-    arys.shift while !arys.empty? && arys.first.all? {|v| v.nil? || v == '' }
-    if !arys.empty?
-      while arys.all? {|a| a.empty? || (a.first.nil? || a.first == '') }
-        arys.each {|a| a.shift }
-      end
-    end
-    with_table_stream_output {|gen|
-      arys.each {|a| gen << a }
+    with_output {|out|
+      er.write_to_csv(out, false)
     }
   end
 end