tb 0.9 → 1.0

data/lib/tb/ropen.rb

@@ -1,6 +1,6 @@
-# lib/tb/ropen.rb - Tb::Reader.open
+# lib/tb/ropen.rb - Tb.open_reader
 #
-# Copyright (C) 2011-2013 Tanaka Akira  <akr@fsij.org>
+# Copyright (C) 2011-2014 Tanaka Akira  <akr@fsij.org>
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -28,70 +28,65 @@
 # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
 # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-def Tb.open_reader(filename, opts={})
-  opts = opts.dup
-  case filename
-  when /\Acsv:/
-    filename = $'
-    table_reader_maker = lambda {|io| Tb::CSVReader.new(io) }
-  when /\Atsv:/
-    filename = $'
-    table_reader_maker = lambda {|io| Tb::TSVReader.new(io) }
-  when /\Altsv:/
-    filename = $'
-    pairs_reader_maker = lambda {|io| Tb::LTSVReader.new(io) }
-  when /\Ap[pgbn]m:/
-    filename = $'
-    table_reader_maker = lambda {|io| Tb.pnm_stream_input(io) }
-  when /\Ajson:/
-    filename = $'
-    whole_reader_maker = lambda {|io| Tb::JSONReader.new(io.read) }
-  when /\.csv\z/
-    table_reader_maker = lambda {|io| Tb::CSVReader.new(io) }
-  when /\.tsv\z/
-    table_reader_maker = lambda {|io| Tb::TSVReader.new(io) }
-  when /\.ltsv\z/
-    pairs_reader_maker = lambda {|io| Tb::LTSVReader.new(io) }
-  when /\.p[pgbn]m\z/
-    table_reader_maker = lambda {|io| Tb.pnm_stream_input(io) }
-  when /\.json\z/
-    whole_reader_maker = lambda {|io| Tb::JSONReader.new(io.read) }
+Tb::FormatHash = {
+  'csv'    => { :reader => Tb::HeaderCSVReader,  :writer => Tb::HeaderCSVWriter},
+  'ncsv'   => { :reader => Tb::NumericCSVReader, :writer => Tb::NumericCSVWriter},
+  'tsv'    => { :reader => Tb::HeaderTSVReader,  :writer => Tb::HeaderTSVWriter},
+  'ntsv'   => { :reader => Tb::NumericTSVReader, :writer => Tb::NumericTSVWriter},
+  'ltsv'   => { :reader => Tb::LTSVReader,       :writer => Tb::LTSVWriter},
+  'pnm'    => { :reader => Tb::PNMReader,        :writer => Tb::PNMWriter},
+  'ppm'    => { :reader => Tb::PNMReader,        :writer => Tb::PNMWriter},
+  'pgm'    => { :reader => Tb::PNMReader,        :writer => Tb::PNMWriter},
+  'pbm'    => { :reader => Tb::PNMReader,        :writer => Tb::PNMWriter},
+  'json'   => { :reader => Tb::JSONReader,       :writer => Tb::JSONWriter},
+  'ndjson' => { :reader => Tb::NDJSONReader,     :writer => Tb::NDJSONWriter},
+}
+
+def Tb.undecorate_filename(filename, numeric)
+  if filename.respond_to?(:to_str)
+    filename = filename.to_str
+  elsif filename.respond_to?(:to_path)
+    filename = filename.to_path
   else
-    table_reader_maker = lambda {|io| Tb::CSVReader.new(io) }
-  end
-  if !filename.respond_to?(:to_str) && !filename.respond_to?(:to_path)
     raise ArgumentError, "unexpected filename: #{filename.inspect}"
   end
-  if whole_reader_maker
-    if filename == '-'
-      reader = whole_reader_maker.call($stdin)
-    else
-      reader = File.open(filename) {|io|
-        whole_reader_maker.call(io)
-      }
-    end
-  elsif pairs_reader_maker
-    if filename == '-'
-      reader = pairs_reader_maker.call($stdin)
-    else
-      reader = pairs_reader_maker.call(File.open(filename))
+  if /\A([a-z0-9]{2,}):/ =~ filename
+    fmt = $1
+    filename = $'
+    err("unexpected format: #{fmt.inspect}") if !Tb::FormatHash.has_key?(fmt)
+  elsif /\.([a-z0-9]+{2,})\z/ =~ filename
+    fmt = $1
+    fmt = 'csv' if !Tb::FormatHash.has_key?(fmt)
+  else
+    fmt = 'csv'
+  end
+  if numeric
+    case fmt
+    when 'csv' then fmt = 'ncsv'
+    when 'tsv' then fmt = 'ntsv'
     end
+  end
+  return filename, fmt
+end
+
+def Tb.open_reader(filename, numeric=false)
+  filename, fmt = Tb.undecorate_filename(filename, numeric)
+  factory = Tb::FormatHash.fetch(fmt)[:reader]
+  io_opened = nil
+  if filename == '-'
+    reader = factory.new($stdin)
   else
-    # table_reader_maker should be available.
-    reader = Tb::Reader.new(opts) {|body|
-      if filename == '-'
-        rawreader = table_reader_maker.call($stdin)
-        body.call(rawreader)
-      else
-        File.open(filename) {|io|
-          rawreader = table_reader_maker.call(io)
-          body.call(rawreader)
-        }
-      end
-    }
+    io_opened = File.open(filename)
+    reader = factory.new(io_opened)
   end
   if block_given?
-    yield reader
+    begin
+      yield reader
+    ensure
+      if io_opened && !io_opened.closed?
+        io_opened.close
+      end
+    end
   else
     reader
   end

data/lib/tb/tsv.rb

@@ -1,6 +1,6 @@
 # lib/tb/tsv.rb - TSV related fetures for table library
 #
-# Copyright (C) 2010-2012 Tanaka Akira  <akr@fsij.org>
+# Copyright (C) 2010-2014 Tanaka Akira  <akr@fsij.org>
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -30,89 +30,57 @@
 
 require 'stringio'
 
-class Tb
-  def Tb.load_tsv(filename, *header_fields, &block)
-    Tb.parse_tsv(File.read(filename), *header_fields, &block)
+module Tb
+  def Tb.tsv_fields_join(values)
+    values.map {|v| v.to_s.gsub(/[\t\r\n]/, ' ') }.join("\t") + "\n"
   end
 
-  def Tb.parse_tsv(tsv, *header_fields)
-    aa = []
-    tsv_stream_input(tsv) {|ary|
-      aa << ary
-    }
-    aa = yield aa if block_given?
-    if header_fields.empty?
-      reader = Tb::Reader.new {|body| body.call(aa) }
-      reader.to_tb
-    else
-      header = header_fields
-      arys = aa
-      t = Tb.new(header)
-      arys.each {|ary|
-        ary << nil while ary.length < header.length
-        t.insert_values header, ary
-      }
-      t
-    end
+  def Tb.tsv_fields_split(line)
+    line = line.chomp("\n")
+    line = line.chomp("\r")
+    line.split(/\t/, -1)
   end
 
-  def Tb.tsv_stream_input(tsv)
-    tsvreader = TSVReader.new(tsv)
-    while ary = tsvreader.shift
-      yield ary
+  class HeaderTSVReader < HeaderReader
+    def initialize(io)
+      super lambda {
+        line = io.gets
+        if line
+          Tb.tsv_fields_split(line)
+        else
+          nil
+        end
+      }
     end
-    nil
   end
 
-  class TSVReader
-    def initialize(input)
-      if input.respond_to? :to_str
-        @input = StringIO.new(input)
-      else
-        @input = input
-      end
-    end
-
-    def shift
-      line = @input.gets
-      return nil if !line
-      line = line.chomp("\n")
-      line = line.chomp("\r")
-      line.split(/\t/, -1)
+  class HeaderTSVWriter < HeaderWriter
+    # io is an object which has "<<" method.
+    def initialize(io)
+      super lambda {|ary|
+        io << Tb.tsv_fields_join(ary)
+      }
     end
   end
 
-  def Tb.tsv_stream_output(out)
-    gen = Object.new
-    gen.instance_variable_set(:@out, out)
-    def gen.<<(ary)
-      @out << Tb.tsv_fields_join(ary) << "\n"
+  class NumericTSVReader < NumericReader
+    def initialize(io)
+      super lambda {
+        line = io.gets
+        if line
+          Tb.tsv_fields_split(line)
+        else
+          nil
+        end
+      }
     end
-    yield gen
   end
 
-  # :call-seq:
-  #   generate_tsv(out='', fields=nil) {|recordids| modified_recordids }
-  #   generate_tsv(out='', fields=nil)
-  #
-  def generate_tsv(out='', fields=nil, &block)
-    if fields.nil?
-      fields = list_fields
-    end
-    recordids = list_recordids
-    if block_given?
-      recordids = yield(recordids)
-    end
-    Tb.tsv_stream_output(out) {|gen|
-      gen << fields
-      recordids.each {|recordid|
-        gen << get_values(recordid, *fields)
+  class NumericTSVWriter < NumericWriter
+    def initialize(io)
+      super lambda {|ary|
+        io << Tb.tsv_fields_join(ary)
       }
-    }
-    out
-  end
-
-  def Tb.tsv_fields_join(values)
-    values.map {|v| v.to_s.gsub(/[\t\r\n]/, ' ') }.join("\t")
+    end
   end
 end

data/sample/excel2csv

@@ -2,7 +2,7 @@
 
 # sample/excel2csv - XLS to CSV converter using Microsoft Excel via WIN32OLE.
 #
-# Copyright (C) 2011-2012 Tanaka Akira  <akr@fsij.org>
+# Copyright (C) 2011-2014 Tanaka Akira  <akr@fsij.org>
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -205,7 +205,7 @@ end
 def generate_csv_row(csvgen, row)
   row = row.dup
   row.pop while !row.empty? && row.last.nil?
-  csvgen << row
+  csvgen.call(row)
 end
 
 def convert_sheet(filename, sheet, csvgen)
@@ -243,30 +243,29 @@ excel = WIN32OLE.new('Excel.Application')
 WIN32OLE.const_load(excel,Excel)
 begin
   excel.displayAlerts = false
-  Tb.csv_stream_output($stdout) {|csvgen|
-    ARGV.each {|excel_filename|
-      if File.directory? excel_filename
-        filenames = []
-        Find.find(excel_filename) {|path| filenames << path if /\.xls\z/ =~ path }
-      else
-        filenames = [excel_filename]
-      end
-      filenames.each {|ifn0|
-        ifn = getAbsolutePath(ifn0)
-        wb = excel.Workbooks.Open(ifn)
-        begin
-          if $opt_all_sheets
-            wb.Worksheets.each {|sheet|
-              sheetname = sheet.Name
-              convert_sheet(ifn0, sheet, csvgen)
-            }
-          else
-            convert_sheet(ifn0, wb.Worksheets(1), csvgen)
-          end
-        ensure
-          wb.Close
+  csvgen = lambda {|row| $stdout << row.to_csv }
+  ARGV.each {|excel_filename|
+    if File.directory? excel_filename
+      filenames = []
+      Find.find(excel_filename) {|path| filenames << path if /\.xls\z/ =~ path }
+    else
+      filenames = [excel_filename]
+    end
+    filenames.each {|ifn0|
+      ifn = getAbsolutePath(ifn0)
+      wb = excel.Workbooks.Open(ifn)
+      begin
+        if $opt_all_sheets
+          wb.Worksheets.each {|sheet|
+            sheetname = sheet.Name
+            convert_sheet(ifn0, sheet, csvgen)
+          }
+        else
+          convert_sheet(ifn0, wb.Worksheets(1), csvgen)
         end
-      }
+      ensure
+        wb.Close
+      end
     }
   }
 ensure

data/sample/poi-xls2csv.rb

@@ -2,7 +2,7 @@
 
 # sample/poi-xls2csv.rb - XLS to CSV convert using Apache POI with JRuby.
 #
-# Copyright (C) 2011-2012 Tanaka Akira  <akr@fsij.org>
+# Copyright (C) 2011-2014 Tanaka Akira  <akr@fsij.org>
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -354,7 +354,7 @@ def convert_sheet(filename, book, i, csvgen)
     sheetname += ":sheetname" if $opt_type
     sheet_header << sheetname
   end
-  csvgen << (sheet_header + convert_horizontal_borders(sheet, merged, rownums.first-1, min_firstcol)) if $opt_border
+  csvgen.call (sheet_header + convert_horizontal_borders(sheet, merged, rownums.first-1, min_firstcol)) if $opt_border
   rownums.each {|y|
     record = []
     row = sheet.getRow(y)
@@ -370,8 +370,8 @@ def convert_sheet(filename, book, i, csvgen)
         }
       end
     end
-    csvgen << (sheet_header + record)
-    csvgen << (sheet_header + convert_horizontal_borders(sheet, merged, y, min_firstcol)) if $opt_border
+    csvgen.call (sheet_header + record)
+    csvgen.call (sheet_header + convert_horizontal_borders(sheet, merged, y, min_firstcol)) if $opt_border
   }
 end
 
@@ -386,16 +386,15 @@ def convert_book(filename, input, csvgen)
   end
 end
 
-Tb.csv_stream_output($stdout) {|csvgen|
-  argv = ARGV.empty? ? ['-'] : ARGV
-  argv.each {|filename|
-    if filename == '-'
-      input = java.lang.System.in
-    else
-      input = java.io.FileInputStream.new(filename)
-    end
-    convert_book(filename, input, csvgen)
-  }
+csvgen = lambdas {|row| $stdout << row.to_csv }
+argv = ARGV.empty? ? ['-'] : ARGV
+argv.each {|filename|
+  if filename == '-'
+    input = java.lang.System.in
+  else
+    input = java.io.FileInputStream.new(filename)
+  end
+  convert_book(filename, input, csvgen)
 }
 
 exit true

data/tb.gemspec

@@ -0,0 +1,154 @@
+Gem::Specification.new do |s|
+  s.name = 'tb'
+  s.version = '1.0'
+  s.date = '2014-10-29'
+  s.author = 'Tanaka Akira'
+  s.email = 'akr@fsij.org'
+  s.required_ruby_version = '>= 1.9.2'
+  s.add_development_dependency 'test-unit', '~> 2.5.5'
+  s.files = %w[
+    README
+    bin/tb
+    lib/tb.rb
+    lib/tb/catreader.rb
+    lib/tb/cmd_cat.rb
+    lib/tb/cmd_consecutive.rb
+    lib/tb/cmd_crop.rb
+    lib/tb/cmd_cross.rb
+    lib/tb/cmd_cut.rb
+    lib/tb/cmd_git.rb
+    lib/tb/cmd_group.rb
+    lib/tb/cmd_gsub.rb
+    lib/tb/cmd_help.rb
+    lib/tb/cmd_join.rb
+    lib/tb/cmd_ls.rb
+    lib/tb/cmd_melt.rb
+    lib/tb/cmd_mheader.rb
+    lib/tb/cmd_nest.rb
+    lib/tb/cmd_newfield.rb
+    lib/tb/cmd_rename.rb
+    lib/tb/cmd_search.rb
+    lib/tb/cmd_shape.rb
+    lib/tb/cmd_sort.rb
+    lib/tb/cmd_svn.rb
+    lib/tb/cmd_tar.rb
+    lib/tb/cmd_to_csv.rb
+    lib/tb/cmd_to_json.rb
+    lib/tb/cmd_to_ltsv.rb
+    lib/tb/cmd_to_pnm.rb
+    lib/tb/cmd_to_pp.rb
+    lib/tb/cmd_to_tsv.rb
+    lib/tb/cmd_to_yaml.rb
+    lib/tb/cmd_unmelt.rb
+    lib/tb/cmd_unnest.rb
+    lib/tb/cmdmain.rb
+    lib/tb/cmdtop.rb
+    lib/tb/cmdutil.rb
+    lib/tb/csv.rb
+    lib/tb/customcmp.rb
+    lib/tb/customeq.rb
+    lib/tb/enumerable.rb
+    lib/tb/enumerator.rb
+    lib/tb/ex_enumerable.rb
+    lib/tb/ex_enumerator.rb
+    lib/tb/fileenumerator.rb
+    lib/tb/func.rb
+    lib/tb/hashreader.rb
+    lib/tb/hashwriter.rb
+    lib/tb/headerreader.rb
+    lib/tb/headerwriter.rb
+    lib/tb/json.rb
+    lib/tb/ltsv.rb
+    lib/tb/ndjson.rb
+    lib/tb/numericreader.rb
+    lib/tb/numericwriter.rb
+    lib/tb/pager.rb
+    lib/tb/pnm.rb
+    lib/tb/revcmp.rb
+    lib/tb/ropen.rb
+    lib/tb/search.rb
+    lib/tb/tsv.rb
+    lib/tb/zipper.rb
+    sample/colors.ppm
+    sample/excel2csv
+    sample/gradation.pgm
+    sample/langs.csv
+    sample/poi-xls2csv.rb
+    sample/poi-xls2csv.sh
+    sample/tbplot
+    tb.gemspec
+    test-all-cov.rb
+    test-all.rb
+  ]
+  s.test_files = %w[
+    test/test_catreader.rb
+    test/test_cmd_cat.rb
+    test/test_cmd_consecutive.rb
+    test/test_cmd_crop.rb
+    test/test_cmd_cross.rb
+    test/test_cmd_cut.rb
+    test/test_cmd_git_log.rb
+    test/test_cmd_grep.rb
+    test/test_cmd_group.rb
+    test/test_cmd_gsub.rb
+    test/test_cmd_help.rb
+    test/test_cmd_join.rb
+    test/test_cmd_ls.rb
+    test/test_cmd_melt.rb
+    test/test_cmd_mheader.rb
+    test/test_cmd_nest.rb
+    test/test_cmd_newfield.rb
+    test/test_cmd_rename.rb
+    test/test_cmd_shape.rb
+    test/test_cmd_sort.rb
+    test/test_cmd_svn_log.rb
+    test/test_cmd_tar_tvf.rb
+    test/test_cmd_to_csv.rb
+    test/test_cmd_to_json.rb
+    test/test_cmd_to_ltsv.rb
+    test/test_cmd_to_pnm.rb
+    test/test_cmd_to_pp.rb
+    test/test_cmd_to_tsv.rb
+    test/test_cmd_to_yaml.rb
+    test/test_cmd_unmelt.rb
+    test/test_cmd_unnest.rb
+    test/test_cmdtty.rb
+    test/test_cmdutil.rb
+    test/test_csv.rb
+    test/test_customcmp.rb
+    test/test_customeq.rb
+    test/test_ex_enumerable.rb
+    test/test_fileenumerator.rb
+    test/test_headercsv.rb
+    test/test_json.rb
+    test/test_ltsv.rb
+    test/test_ndjson.rb
+    test/test_numericcsv.rb
+    test/test_pager.rb
+    test/test_pnm.rb
+    test/test_reader.rb
+    test/test_revcmp.rb
+    test/test_search.rb
+    test/test_tbenum.rb
+    test/test_tsv.rb
+    test/test_zipper.rb
+    test/util_tbtest.rb
+  ]
+  s.has_rdoc = true
+  s.homepage = 'https://github.com/akr/tb'
+  s.require_path = 'lib'
+  s.executables << 'tb'
+  s.summary = 'manipulation tool for tables: CSV, TSV, LTSV, JSON, NDJSON, etc.'
+  s.description = <<'End'
+tb is a manipulation tool for tables.
+
+tb provides a command and a library for manipulating tables:
+Unix filter like operations (sort, cat, cut, ls, etc.),
+SQL like operations (join, group, etc.),
+other table operations (search, gsub, rename, cross, melt, unmelt, etc.),
+information extractions (git, svn, tar),
+and more.
+
+tb supports various table formats: CSV, TSV, JSON, NDJSON, LTSV, etc.
+End
+end