tb 0.9 → 1.0

data/lib/tb/cmd_to_pnm.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2011-2012 Tanaka Akira  <akr@fsij.org>
+# Copyright (C) 2011-2014 Tanaka Akira  <akr@fsij.org>
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -40,9 +40,9 @@ def (Tb::Cmd).main_to_pnm(argv)
   op_to_pnm.parse!(argv)
   exit_if_help('to-pnm')
   argv = ['-'] if argv.empty?
-  tbl = Tb::CatReader.open(argv, Tb::Cmd.opt_N).to_tb
+  reader = Tb::CatReader.open(argv, Tb::Cmd.opt_N)
   with_output {|out|
-    tbl.generate_pnm(out)
+    reader.write_to_pnm(out)
   }
 end
 

data/lib/tb/cmd_to_tsv.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2011-2012 Tanaka Akira  <akr@fsij.org>
+# Copyright (C) 2011-2014 Tanaka Akira  <akr@fsij.org>
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -40,9 +40,9 @@ def (Tb::Cmd).main_to_tsv(argv)
   op_to_tsv.parse!(argv)
   exit_if_help('to-tsv')
   argv = ['-'] if argv.empty?
-  tbl = Tb::CatReader.open(argv, Tb::Cmd.opt_N).to_tb
+  reader = Tb::CatReader.open(argv, Tb::Cmd.opt_N)
   with_output {|out|
-    tbl_generate_tsv(tbl, out)
+    reader.write_to_tsv(out, !Tb::Cmd.opt_N)
   }
 end
 

data/lib/tb/cmd_to_yaml.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2011-2012 Tanaka Akira  <akr@fsij.org>
+# Copyright (C) 2011-2014 Tanaka Akira  <akr@fsij.org>
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -41,8 +41,8 @@ def (Tb::Cmd).main_to_yaml(argv)
   op_to_yaml.parse!(argv)
   exit_if_help('to-yaml')
   argv = ['-'] if argv.empty?
-  tbl = Tb::CatReader.open(argv, Tb::Cmd.opt_N).to_tb
-  ary = tbl.map {|rec| rec.to_h }
+  reader = Tb::CatReader.open(argv, Tb::Cmd.opt_N)
+  ary = reader.to_a
   with_output {|out|
     YAML.dump(ary, out)
     out.puts

data/lib/tb/cmd_unmelt.rb

@@ -58,6 +58,21 @@ def (Tb::Cmd).op_unmelt
   op
 end
 
+Tb::Cmd.def_vhelp('unmelt', <<'End')
+Example:
+
+  % cat tst.csv
+  foo,variable,value
+  A,bar,1
+  A,baz,x
+  B,bar,2
+  B,baz,y
+  % tb unmelt tst.csv
+  foo,bar,baz
+  A,1,x
+  B,2,y
+End
+
 def (Tb::Cmd).main_unmelt(argv)
   op_unmelt.parse!(argv)
   exit_if_help('unmelt')

data/lib/tb/cmd_unnest.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2011-2012 Tanaka Akira  <akr@fsij.org>
+# Copyright (C) 2011-2014 Tanaka Akira  <akr@fsij.org>
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -41,6 +41,27 @@ def (Tb::Cmd).op_unnest
   op
 end
 
+Tb::Cmd.def_vhelp('unnest', <<'End')
+Example:
+
+  % cat tst.csv
+  author,item
+  A,"name,length
+  foo,3
+  bar,5
+  "
+  B,"name,length
+  baz,2
+  qux,8
+  "
+  % tb unnest item tst.csv
+  author,name,length
+  A,foo,3
+  A,bar,5
+  B,baz,2
+  B,qux,8
+End
+
 def (Tb::Cmd).main_unnest(argv)
   op_unnest.parse!(argv)
   exit_if_help('unnest')
@@ -64,13 +85,16 @@ def (Tb::Cmd).main_unnest(argv)
           elsif v.nil?
             pairs2[f] = v
           else
-            nested_tbl = Tb.parse_csv(v)
-            nested_tbl.list_fields.each {|f2|
+            aa = CSV.parse(v)
+            reader = Tb::HeaderReader.new(lambda { aa.shift })
+            nested_tbl_rows = reader.to_a
+            nested_tbl_header = reader.get_named_header
+            nested_tbl_header.each {|f2|
               unless nested_fields.has_key? f2
                 nested_fields[f2] = nested_fields.size
               end
             }
-            pairs2[f] = nested_tbl
+            pairs2[f] = [nested_tbl_header, nested_tbl_rows]
           end
         }
         y2.yield pairs2
@@ -93,13 +117,13 @@ def (Tb::Cmd).main_unnest(argv)
     }.each {|pairs|
       pairs2 = {}
       pairs.each {|f, v| pairs2[f] = v if f != target_field }
-      ntbl = pairs[target_field]
-      if ntbl.nil? || ntbl.empty?
+      ntbl_header, ntbl_rows = pairs[target_field]
+      if ntbl_header.nil? || ntbl_rows.empty?
         if Tb::Cmd.opt_unnest_outer
           y.yield pairs2
         end
       else
-        ntbl.each {|npairs|
+        ntbl_rows.each {|npairs|
           pairs3 = pairs2.dup
           npairs.each {|nf, nv|
             if Tb::Cmd.opt_unnest_prefix

data/lib/tb/cmdmain.rb

@@ -45,5 +45,7 @@ def (Tb::Cmd).main(argv)
 rescue SystemExit
   $stderr.puts $!.message if $!.message != 'exit'
   raise
+rescue Errno::EPIPE
+  exit false
 end
 

data/lib/tb/cmdtop.rb

@@ -43,7 +43,7 @@ require 'tb/cmd_to_pnm'
 require 'tb/cmd_to_json'
 require 'tb/cmd_to_yaml'
 require 'tb/cmd_to_pp'
-require 'tb/cmd_grep'
+require 'tb/cmd_search'
 require 'tb/cmd_gsub'
 require 'tb/cmd_sort'
 require 'tb/cmd_cut'

data/lib/tb/cmdutil.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2011-2013 Tanaka Akira  <akr@fsij.org>
+# Copyright (C) 2011-2014 Tanaka Akira  <akr@fsij.org>
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -131,39 +131,11 @@ def split_field_list_argument(arg)
 end
 
 def split_csv_argument(arg)
-  Tb.csv_stream_input(arg) {|ary| return ary }
-  return []
+  return CSV.new(arg).shift || []
 end
 
 def tablereader_open(filename, &b)
-  Tb.open_reader(filename, {:numeric=>Tb::Cmd.opt_N}, &b)
-end
-
-def tbl_generate_tsv(tbl, out)
-  if Tb::Cmd.opt_N
-    header = tbl.list_fields
-    Tb.tsv_stream_output(out) {|gen|
-      tbl.each {|rec|
-        gen << rec.values_at(*header)
-      }
-    }
-  else
-    tbl.generate_tsv(out)
-  end
-end
-
-def tbl_generate_ltsv(tbl, out)
-  if Tb::Cmd.opt_N
-    header = tbl.list_fields
-    Tb.ltsv_stream_output(out) {|gen|
-      tbl.each {|rec|
-        assoc = header.map {|f| [f, rec[f]] }
-        gen << assoc
-      }
-    }
-  else
-    tbl.generate_ltsv(out)
-  end
+  Tb.open_reader(filename, Tb::Cmd.opt_N, &b)
 end
 
 def with_output(filename=Tb::Cmd.opt_output)
@@ -191,37 +163,12 @@ def with_output(filename=Tb::Cmd.opt_output)
 end
 
 def output_tbenum(te)
-  filename = Tb::Cmd.opt_output
-  if /\A([a-z0-9]{2,}):/ =~ filename
-    fmt = $1
-    filename = $'
-  else
-    fmt = nil
-  end
-  if !fmt
-    case filename
-    when /\.csv\z/
-      fmt = 'csv'
-    when /\.ltsv\z/
-      fmt = 'ltsv'
-    when /\.json\z/
-      fmt = 'json'
-    end
-  end
-  if fmt
-    case fmt
-    when 'csv'
-      write_proc = lambda {|out| te.write_to_csv(out, !Tb::Cmd.opt_N) }
-    when 'ltsv'
-      write_proc = lambda {|out| te.write_to_ltsv(out) }
-    when 'json'
-      write_proc = lambda {|out| te.write_to_json(out) }
-    else
-      err("unexpected format: #{fmt.inspect}")
-    end
-  end
-  write_proc ||= lambda {|out| te.write_to_csv(out, !Tb::Cmd.opt_N) }
+  filename = Tb::Cmd.opt_output || '-'
+  numeric = Tb::Cmd.opt_N
+  filename, fmt = Tb.undecorate_filename(filename, numeric)
+  factory = Tb::FormatHash.fetch(fmt)[:writer]
   with_output(filename) {|out|
-    write_proc.call(out)
+    writer = factory.new(out)
+    te.write_with(writer)
   }
 end

data/lib/tb/csv.rb

@@ -1,6 +1,6 @@
 # lib/tb/csv.rb - CSV related fetures for table library
 #
-# Copyright (C) 2010-2013 Tanaka Akira  <akr@fsij.org>
+# Copyright (C) 2010-2014 Tanaka Akira  <akr@fsij.org>
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -30,95 +30,37 @@
 
 require 'csv'
 
-class Tb
-  def Tb.load_csv(filename, *header_fields, &block)
-    Tb.parse_csv(File.read(filename), *header_fields, &block)
+module Tb
+  def Tb.csv_encode_row(ary)
+    ary.to_csv
   end
 
-  def Tb.parse_csv(csv, *header_fields)
-    aa = []
-    csv_stream_input(csv) {|ary|
-      aa << ary
-    }
-    aa = yield aa if block_given?
-    if header_fields.empty?
-      reader = Tb::Reader.new {|body| body.call(aa) }
-      reader.to_tb
-    else
-      header = header_fields
-      arys = aa
-      t = Tb.new(header)
-      arys.each {|ary|
-        ary << nil while ary.length < header.length
-        t.insert_values header, ary
-      }
-      t
+  class HeaderCSVReader < HeaderReader
+    def initialize(io)
+      aryreader = CSV.new(io)
+      super lambda { aryreader.shift }
     end
   end
 
-  def Tb.csv_stream_input(csv, &b)
-    csvreader = CSVReader.new(csv)
-    csvreader.each(&b)
-    nil
-  end
-
-  def Tb.csv_read_aa(csv)
-    aa = []
-    Tb.csv_stream_input(csv) {|ary| aa << ary }
-    aa
-  end
-
-  class CSVReader
-    def initialize(input)
-      @csv = CSV.new(input)
-    end
-
-    def shift
-      @csv.shift
-    end
-
-    def each
-      while ary = self.shift
-        yield ary
-      end
-      nil
+  class HeaderCSVWriter < HeaderWriter
+    # io is an object which has "<<" method.
+    def initialize(io)
+      super lambda {|ary| io << ary.to_csv}
     end
   end
 
-  def Tb.csv_stream_output(out)
-    require 'csv'
-    gen = Object.new
-    gen.instance_variable_set(:@out, out)
-    def gen.<<(ary)
-      @out << ary.to_csv
+  class NumericCSVReader < NumericReader
+    def initialize(io)
+      aryreader = CSV.new(io)
+      super lambda { aryreader.shift }
     end
-    yield gen
-  end
-
-  def Tb.csv_encode_row(ary)
-    require 'csv'
-    ary.to_csv
   end
 
-  # :call-seq:
-  #   generate_csv(out='', fields=nil) {|recordids| modified_recordids }
-  #   generate_csv(out='', fields=nil)
-  #
-  def generate_csv(out='', fields=nil, &block)
-    if fields.nil?
-      fields = list_fields
+  class NumericCSVWriter < NumericWriter
+    # io is an object which has "<<" method.
+    def initialize(io)
+      super lambda {|ary| io << ary.to_csv }
     end
-    require 'csv'
-    recordids = list_recordids
-    if block_given?
-      recordids = yield(recordids)
-    end
-    Tb.csv_stream_output(out) {|gen|
-      gen << fields
-      recordids.each {|recordid|
-        gen << get_values(recordid, *fields)
-      }
-    }
-    out
   end
+
 end

data/lib/tb/enumerable.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2012-2013 Tanaka Akira  <akr@fsij.org>
+# Copyright (C) 2012-2014 Tanaka Akira  <akr@fsij.org>
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -93,7 +93,7 @@ module Tb::Enumerable
   # creates a new Tb::Enumerator object which have
   # new field named by _field_ with the value returned by the block.
   #
-  #   t1 = Tb.new %w[a b], [1, 2], [3, 4]
+  #   t1 = Tb::Enumerator.from_header_and_values %w[a b], [1, 2], [3, 4]
   #   p t1.newfield("x") {|row| row["a"] + row["b"] + 100 }.to_a
   #   #=> [{"x"=>103, "a"=>1, "b"=>2},
   #   #    {"x"=>107, "a"=>3, "b"=>4}]
@@ -102,11 +102,11 @@ module Tb::Enumerable
     Tb::Enumerator.new {|y|
       self.with_header {|header|
         if header
-          y.set_header(Tb::FieldSet.normalize([field, *header]))
+          y.set_header([field, *header])
         end
       }.each {|row|
         keys = row.keys
-        keys = Tb::FieldSet.normalize([field, *keys])
+        keys = [field, *keys]
         vals = row.values
         vals = [yield(row), *vals]
         y << Hash[keys.zip(vals)]
@@ -176,80 +176,46 @@ module Tb::Enumerable
     natjoin2(tbl2, missing_value, retain_left, retain_right)
   end
 
-  def to_tb
-    tb = Tb.new
-    self.each {|pairs|
-      pairs.each {|k, v|
-        unless tb.has_field? k
-          tb.define_field(k)
-        end
+  def write_with(writer)
+    header_proc = nil
+    if writer.header_required?
+      header_proc = lambda {|header|
+        writer.header_generator = lambda { header }
       }
-      tb.insert pairs
+    end
+    body_proc = lambda  {|pairs|
+      writer.put_hash pairs
     }
-    tb
+    header_and_each(header_proc, &body_proc)
+    writer.finish
   end
 
   def write_to_csv(io, with_header=true)
-    stream = nil
-    header = []
-    fgen = fnew = nil
-    self.with_cumulative_header {|header0|
-      if !with_header
-        stream = true
-      elsif header0
-        stream = true
-        io.puts Tb.csv_encode_row(header0)
-      else
-        stream = false
-        fgen, fnew = Tb::FileEnumerator.gen_new
-      end
-    }.each {|pairs, header1|
-      pairs = Hash[pairs] unless pairs.respond_to? :has_key?
-      header = header1
-      if stream
-        fs = header.dup
-        while !fs.empty? && !pairs.has_key?(fs.last)
-          fs.pop
-        end
-        ary = fs.map {|f| pairs[f] }
-        io.puts Tb.csv_encode_row(ary)
-      else
-        fgen.call Hash[pairs]
-      end
-    }
-    if !stream
-      if with_header
-        io.puts Tb.csv_encode_row(header)
-      end
-      fnew.call.each {|pairs|
-        fs = header.dup
-        while !fs.empty? && !pairs.has_key?(fs.last)
-          fs.pop
-        end
-        ary = fs.map {|f| pairs[f] }
-        io.puts Tb.csv_encode_row(ary)
-      }
+    if with_header
+      write_with(Tb::HeaderCSVWriter.new(io))
+    else
+      write_with(Tb::NumericCSVWriter.new(io))
     end
   end
 
-  def write_to_ltsv(out)
-    self.each {|pairs|
-      out.print Tb.ltsv_assoc_join(pairs)
-      out.print "\n"
-    }
+  def write_to_tsv(io, with_header=true)
+    if with_header
+      write_with(Tb::HeaderTSVWriter.new(io))
+    else
+      write_with(Tb::NumericTSVWriter.new(io))
+    end
   end
 
-  def write_to_json(out)
-    require 'json'
-    out.print "["
-    sep = nil
-    self.each {|pairs|
-      out.print sep if sep
-      out.print JSON.pretty_generate(Hash[pairs.to_a])
-      sep = ",\n"
-    }
-    out.puts "]"
-    nil
+  def write_to_ltsv(io)
+    write_with(Tb::LTSVWriter.new(io))
+  end
+
+  def write_to_json(io)
+    write_with(Tb::JSONWriter.new(io))
+  end
+
+  def write_to_pnm(io)
+    write_with(Tb::PNMWriter.new(io))
   end
 
   def extsort_by(opts={}, &cmpvalue_from)
@@ -270,3 +236,11 @@ module Tb::Enumerable
     }
   end
 end
+
+module Tb::EnumerableWithEach
+  include Tb::Enumerable
+
+  def each(&b)
+    header_and_each(nil, &b)
+  end
+end