tb 0.1 → 0.2

data/lib/tb.rb

@@ -28,8 +28,10 @@ require 'tb/basic'
 require 'tb/record'
 require 'tb/csv'
 require 'tb/tsv'
-require 'tb/qtsv'
+require 'tb/pnm'
 require 'tb/reader'
+require 'tb/ropen'
+require 'tb/catreader'
 require 'tb/fieldset'
-require 'tb/pathfinder'
+require 'tb/search'
 require 'tb/enumerable'

data/lib/tb/catreader.rb

@@ -0,0 +1,131 @@
+# lib/tb/catreader.rb - Tb::CatReader class
+#
+# Copyright (C) 2011 Tanaka Akira  <akr@fsij.org>
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+#  1. Redistributions of source code must retain the above copyright notice, this
+#     list of conditions and the following disclaimer.
+#  2. Redistributions in binary form must reproduce the above copyright notice,
+#     this list of conditions and the following disclaimer in the documentation
+#     and/or other materials provided with the distribution.
+#  3. The name of the author may not be used to endorse or promote products
+#     derived from this software without specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+# EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+# OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+# OF SUCH DAMAGE.
+
+class Tb::CatReader
+  def self.open(filenames, numeric=false)
+    readers = []
+    filenames.each {|f|
+      readers << Tb::Reader.open(f, numeric ? {:numeric=>true} : {})
+    }
+    r = Tb::CatReader.new(readers, readers, numeric)
+    if block_given?
+      begin
+        yield r
+      ensure
+        r.close
+      end
+    else
+      r
+    end
+  end
+
+  def initialize(readers, also_close, numeric)
+    @readers = readers.dup
+    @also_close = also_close
+    @numeric = numeric
+    @fieldset = nil
+  end
+
+  def header
+    return @fieldset.header if @fieldset
+    if @numeric
+      @fieldset = Tb::FieldSet.new
+    else
+      h = {}
+      @readers.each {|r|
+        r.header.each {|f|
+          if !h[f]
+            h[f] = h.size
+          end
+        }
+      }
+      @fieldset = Tb::FieldSet.new(*h.keys.sort_by {|f| h[f] })
+    end
+    return @fieldset.header
+  end
+
+  def index_from_field_ex(f)
+    self.header
+    @fieldset.index_from_field_ex(f)
+  end
+
+  def index_from_field(f)
+    self.header
+    @fieldset.index_from_field(f)
+  end
+
+  def field_from_index_ex(i)
+    self.header
+    @fieldset.field_from_index_ex(i)
+  end
+
+  def field_from_index(i)
+    self.header
+    @fieldset.field_from_index(i)
+  end
+
+  def shift
+    self.header
+    while !@readers.empty?
+      r = @readers.first
+      ary = r.shift
+      if ary
+        h = r.header
+        ary2 = []
+        ary.each_with_index {|v,i|
+          f = h[i]
+          i2 = @fieldset.index_from_field_ex(f)
+          ary2[i2] = v
+        }
+        return ary2
+      else
+        @readers.shift
+      end
+    end
+    nil
+  end
+
+  def each
+    while ary = self.shift
+      yield ary
+    end
+    nil
+  end
+
+  def read_all
+    result = []
+    while ary = self.shift
+      result << ary
+    end
+    result
+  end
+
+  def close
+    if @also_close
+      @also_close.each {|x| x.close }
+    end
+  end
+end

data/lib/tb/cmd_cat.rb

@@ -0,0 +1,65 @@
+# Copyright (C) 2011 Tanaka Akira  <akr@fsij.org>
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+#  1. Redistributions of source code must retain the above copyright notice, this
+#     list of conditions and the following disclaimer.
+#  2. Redistributions in binary form must reproduce the above copyright notice,
+#     this list of conditions and the following disclaimer in the documentation
+#     and/or other materials provided with the distribution.
+#  3. The name of the author may not be used to endorse or promote products
+#     derived from this software without specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+# EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+# OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+# OF SUCH DAMAGE.
+
+Tb::Cmd.subcommands << 'cat'
+
+def (Tb::Cmd).op_cat
+  op = OptionParser.new
+  op.banner = 'Usage: tb cat [OPTS] [TABLE ...]'
+  define_common_option(op, "hNo", "--no-pager")
+  op
+end
+
+Tb::Cmd.def_vhelp('cat', <<'End')
+Example:
+
+  % cat tst1.csv 
+  a,b,c
+  0,1,2
+  4,5,6
+  % cat tst2.csv 
+  a,b,d
+  U,V,W
+  X,Y,Z
+  % tb cat tst1.csv tst2.csv
+  a,b,c,d
+  0,1,2
+  4,5,6
+  U,V,,W
+  X,Y,,Z
+End
+
+def (Tb::Cmd).main_cat(argv)
+  op_cat.parse!(argv)
+  exit_if_help('cat')
+  argv = ['-'] if argv.empty?
+  creader = Tb::CatReader.open(argv, Tb::Cmd.opt_N)
+  header = creader.header
+  with_table_stream_output {|gen|
+    gen << header if !Tb::Cmd.opt_N
+    creader.each {|ary|
+      gen << ary
+    }
+  }
+end

data/lib/tb/cmd_consecutive.rb

@@ -0,0 +1,79 @@
+# Copyright (C) 2011 Tanaka Akira  <akr@fsij.org>
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+#  1. Redistributions of source code must retain the above copyright notice, this
+#     list of conditions and the following disclaimer.
+#  2. Redistributions in binary form must reproduce the above copyright notice,
+#     this list of conditions and the following disclaimer in the documentation
+#     and/or other materials provided with the distribution.
+#  3. The name of the author may not be used to endorse or promote products
+#     derived from this software without specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+# EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+# OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+# OF SUCH DAMAGE.
+
+Tb::Cmd.subcommands << 'consecutive'
+
+Tb::Cmd.default_option[:opt_consecutive_n] = 2
+
+def (Tb::Cmd).op_consecutive
+  op = OptionParser.new
+  op.banner = 'Usage: tb consecutive [OPTS] [TABLE ...]'
+  define_common_option(op, "hNo", "--no-pager")
+  op.def_option('-n NUM', 'gather NUM records.  (default: 2)') {|n| Tb::Cmd.opt_consecutive_n = n.to_i }
+  op
+end
+
+Tb::Cmd.def_vhelp('consecutive', <<'End')
+Example:
+
+  % cat tst.csv 
+  a,b,c
+  0,1,2
+  4,5,6
+  7,8,9
+  % tb consecutive tstcsv
+  a_1,a_2,b_1,b_2,c_1,c_2
+  0,4,1,5,2,6
+  4,7,5,8,6,9
+End
+
+def (Tb::Cmd).main_consecutive(argv)
+  op_consecutive.parse!(argv)
+  exit_if_help('consecutive')
+  argv = ['-'] if argv.empty?
+  creader = Tb::CatReader.open(argv, Tb::Cmd.opt_N)
+  consecutive_header = []
+  creader.header.each {|f|
+    Tb::Cmd.opt_consecutive_n.times {|i|
+      consecutive_header << "#{f}_#{i+1}"
+    }
+  }
+  with_table_stream_output {|gen|
+    gen.output_header consecutive_header
+    buf = []
+    creader.each {|ary|
+      buf << ary
+      if buf.length == Tb::Cmd.opt_consecutive_n
+        ary2 = []
+        buf.each_with_index {|a, i|
+          a.each_with_index {|e, j|
+            ary2[j*Tb::Cmd.opt_consecutive_n + i] = e
+          }
+        }
+        gen << ary2
+        buf.shift
+      end
+    }
+  }
+end

data/lib/tb/cmd_crop.rb

@@ -0,0 +1,105 @@
+# Copyright (C) 2011 Tanaka Akira  <akr@fsij.org>
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+#  1. Redistributions of source code must retain the above copyright notice, this
+#     list of conditions and the following disclaimer.
+#  2. Redistributions in binary form must reproduce the above copyright notice,
+#     this list of conditions and the following disclaimer in the documentation
+#     and/or other materials provided with the distribution.
+#  3. The name of the author may not be used to endorse or promote products
+#     derived from this software without specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+# EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+# OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+# OF SUCH DAMAGE.
+
+Tb::Cmd.subcommands << 'crop'
+
+Tb::Cmd.default_option[:opt_crop_range] = nil
+
+def (Tb::Cmd).op_crop
+  op = OptionParser.new
+  op.banner = 'Usage: tb crop [OPTS] [TABLE ...]'
+  define_common_option(op, "ho", "--no-pager")
+  op.def_option('-r RANGE', 'range.  i.e. "2,1-4,3", "B1:D3"') {|arg| Tb::Cmd.opt_crop_range = arg }
+  op
+end
+
+def (Tb::Cmd).decode_a1_addressing_col(str)
+  (26**str.length-1)/25+str.tr("A-Z", "0-9A-P").to_i(26)
+end
+
+def (Tb::Cmd).main_crop(argv)
+  op_crop.parse!(argv)
+  exit_if_help('crop')
+  argv = ['-'] if argv.empty?
+  stream = false
+  if Tb::Cmd.opt_crop_range
+    case Tb::Cmd.opt_crop_range
+    when /\AR(\d+)C(\d+):R(\d+)C(\d+)\z/ # 1-based (R1C1 reference style)
+      stream = true
+      range_row1 = $1.to_i
+      range_col1 = $2.to_i
+      range_row2 = $3.to_i
+      range_col2 = $4.to_i
+    when /\A([A-Z]+)(\d+):([A-Z]+)(\d+)\z/ # 1-based (A1 reference style)
+      stream = true
+      range_col1 = decode_a1_addressing_col($1)
+      range_row1 = $2.to_i
+      range_col2 = decode_a1_addressing_col($3)
+      range_row2 = $4.to_i
+    else
+      raise ArgumentError, "unexpected range argument: #{Tb::Cmd.opt_crop_range.inspect}"
+    end
+  end
+  if stream
+    with_table_stream_output {|gen|
+      Tb::CatReader.open(argv, true) {|tblreader|
+        rownum = 1
+        tblreader.each {|ary|
+          if range_row2 < rownum
+            break
+          end
+          if range_row1 <= rownum
+            if range_col2 < ary.length
+              ary[range_col2..-1] = []
+            end
+            if 1 < range_col1
+              ary[0...(range_col1-1)] = []
+            end
+            gen << ary
+          end
+          rownum += 1
+        }
+      }
+    }
+  else
+    arys = []
+    Tb::CatReader.open(argv, true) {|tblreader|
+      tblreader.each {|a|
+        a.pop while !a.empty? && (a.last.nil? || a.last == '')
+        arys << a
+      }
+    }
+    arys.pop while !arys.empty? && arys.last.all? {|v| v.nil? || v == '' }
+    arys.shift while !arys.empty? && arys.first.all? {|v| v.nil? || v == '' }
+    if !arys.empty?
+      while arys.all? {|a| a.empty? || (a.first.nil? || a.first == '') }
+        arys.each {|a| a.shift }
+      end
+    end
+    with_table_stream_output {|gen|
+      arys.each {|a| gen << a }
+    }
+  end
+end
+

data/lib/tb/cmd_cross.rb

@@ -0,0 +1,119 @@
+# Copyright (C) 2011 Tanaka Akira  <akr@fsij.org>
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+#  1. Redistributions of source code must retain the above copyright notice, this
+#     list of conditions and the following disclaimer.
+#  2. Redistributions in binary form must reproduce the above copyright notice,
+#     this list of conditions and the following disclaimer in the documentation
+#     and/or other materials provided with the distribution.
+#  3. The name of the author may not be used to endorse or promote products
+#     derived from this software without specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+# EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+# OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+# OF SUCH DAMAGE.
+
+Tb::Cmd.subcommands << 'cross'
+
+Tb::Cmd.default_option[:opt_cross_fields] = []
+Tb::Cmd.default_option[:opt_cross_compact] = false
+
+def (Tb::Cmd).op_cross
+  op = OptionParser.new
+  op.banner = 'Usage: tb cross [OPTS] HKEY-FIELD1,... VKEY-FIELD1,... [TABLE ...]'
+  define_common_option(op, "ho", "--no-pager")
+  op.def_option('-a AGGREGATION-SPEC[,NEW-FIELD]',
+                '--aggregate AGGREGATION-SPEC[,NEW-FIELD]') {|arg| Tb::Cmd.opt_cross_fields << arg }
+  op.def_option('-c', '--compact', 'compact format') { Tb::Cmd.opt_cross_compact = true }
+  op
+end
+
+def (Tb::Cmd).main_cross(argv)
+  op_cross.parse!(argv)
+  exit_if_help('cross')
+  err('no hkey-fields given.') if argv.empty?
+  hkfs = split_field_list_argument(argv.shift)
+  err('no vkey-fields given.') if argv.empty?
+  vkfs = split_field_list_argument(argv.shift)
+  if Tb::Cmd.opt_cross_fields.empty?
+    opt_cross_fields = [['count', 'count']]
+  else
+    opt_cross_fields = Tb::Cmd.opt_cross_fields.map {|arg|
+      agg_spec, new_field = split_field_list_argument(arg)
+      new_field ||= agg_spec
+      [agg_spec, new_field]
+    }
+  end
+  argv = ['-'] if argv.empty?
+  Tb::CatReader.open(argv, Tb::Cmd.opt_N) {|tblreader|
+    vkis = vkfs.map {|f| tblreader.index_from_field(f) }
+    hkis = hkfs.map {|f| tblreader.index_from_field(f) }
+    vset = {}
+    hset = {}
+    set = {}
+    tblreader.each {|ary|
+      vkvs = ary.values_at(*vkis)
+      hkvs = ary.values_at(*hkis)
+      vset[vkvs] = true if !vset.include?(vkvs)
+      hset[hkvs] = true if !hset.include?(hkvs)
+      if !set.include?([vkvs, hkvs])
+        set[[vkvs, hkvs]] = opt_cross_fields.map {|agg_spec, nf|
+          begin
+            ag = make_aggregator(agg_spec, tblreader.header)
+          rescue ArgumentError
+            err($!.message)
+          end
+          ag.update(ary)
+          ag
+        }
+      else
+        set[[vkvs, hkvs]].each {|ag|
+          ag.update(ary)
+        }
+      end
+    }
+    vary = vset.keys.sort_by {|a| a.map {|v| smart_cmp_value(v) } }
+    hary = hset.keys.sort_by {|a| a.map {|v| smart_cmp_value(v) } }
+    with_output {|out|
+      Tb.csv_stream_output(out) {|gen|
+        hkfs.each_with_index {|hkf, i|
+          next if Tb::Cmd.opt_cross_compact && i == hkfs.length - 1
+          row = [nil] * (vkfs.length - 1) + [hkf]
+          hary.each {|hkvs| opt_cross_fields.length.times { row << hkvs[i] } }
+          gen << row
+        }
+        if Tb::Cmd.opt_cross_compact
+          r = vkfs.dup
+          hary.each {|hkvs| r.concat([hkvs[-1]] * opt_cross_fields.length) }
+          gen << r
+        else
+          r = vkfs.dup
+          hary.each {|hkvs| r.concat opt_cross_fields.map {|agg_spec, new_field| new_field } }
+          gen << r
+        end
+        vary.each {|vkvs|
+          row = vkvs.dup
+          hary.each {|hkvs|
+            ags = set[[vkvs, hkvs]]
+            if !ags
+              opt_cross_fields.length.times { row << nil }
+            else
+              ags.each {|ag| row << ag.finish }
+            end
+          }
+          gen << row
+        }
+      }
+    }
+  }
+end
+