tb 0.9 → 1.0

data/test/test_reader.rb

@@ -3,92 +3,44 @@ require 'test/unit'
 require 'tmpdir'
 
 class TestTbReader < Test::Unit::TestCase
-  def test_load_csv
+  def test_open_prefix_csv
     Dir.mktmpdir {|d|
-      open(i="#{d}/i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
+      open(ic="#{d}/c", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
         a,b
         1,3
-        2,4
-      End
-      t = Tb.load_csv(i)
-      records = []
-      t.each_record {|record| records << record.to_h }
-      assert_equal(
-        [{"a"=>"1", "b"=>"3"},
-         {"a"=>"2", "b"=>"4"}],
-        records)
-    }
-  end
-
-  def test_load_tsv
-    Dir.mktmpdir {|d|
-      open(i="#{d}/i.tsv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
-        a\tb
-        1\t3
-        2\t4
       End
-      t = Tb.load_tsv(i)
-      records = []
-      t.each_record {|record| records << record.to_h }
-      assert_equal(
-        [{"a"=>"1", "b"=>"3"},
-         {"a"=>"2", "b"=>"4"}],
-        records)
-    }
-  end
-
-  def test_parse_csv
-    csv = <<-'End'.gsub(/^[ \t]+/, '')
-    1,2
-    3,4
-    End
-    t = Tb.parse_csv(csv, 'a', 'b')
-    records = []
-    t.each_record {|record|
-      records << record.to_h
-    }
-    assert_equal(
-      [{"a"=>"1", "b"=>"2"},
-       {"a"=>"3", "b"=>"4"}],
-      records)
-  end
-
-  def test_parse_tsv
-    csv = <<-"End".gsub(/^[ \t]+/, '')
-    1\t2
-    3\t4
-    End
-    t = Tb.parse_tsv(csv, 'a', 'b')
-    records = []
-    t.each_record {|record|
-      records << record.to_h
+      Tb.open_reader("csv:#{ic}") {|r|
+        header = nil
+        all = []
+        r.with_header {|h| header = h}.each {|pairs| all << pairs.map {|f, v| v } }
+        assert_equal(%w[a b], header)
+        assert_equal([%w[1 3]], all)
+      }
     }
-    assert_equal(
-      [{"a"=>"1", "b"=>"2"},
-       {"a"=>"3", "b"=>"4"}],
-      records)
   end
 
-  def test_open
+  def test_open_no_prefix_suffix_csv
     Dir.mktmpdir {|d|
       open(ic="#{d}/c", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
         a,b
         1,3
       End
-      open(it="#{d}/t", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
-        a\tb
-        1\t3
-      End
-      open(ij="#{d}/j", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
-        [{"a":1,"b":3}]
-      End
-      Tb.open_reader("csv:#{ic}") {|r|
+      Tb.open_reader(ic) {|r|
         header = nil
         all = []
         r.with_header {|h| header = h}.each {|pairs| all << pairs.map {|f, v| v } }
         assert_equal(%w[a b], header)
         assert_equal([%w[1 3]], all)
       }
+    }
+  end
+
+  def test_open_prefix_tsv
+    Dir.mktmpdir {|d|
+      open(it="#{d}/t", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
+        a\tb
+        1\t3
+      End
       Tb.open_reader("tsv:#{it}") {|r|
         header = nil
         all = []
@@ -96,74 +48,25 @@ class TestTbReader < Test::Unit::TestCase
         assert_equal(%w[a b], header)
         assert_equal([%w[1 3]], all)
       }
-      Tb.open_reader("json:#{ij}") {|r|
-        header = nil
-        all = []
-        r.with_header {|h| header = h}.each {|pairs| all << pairs.map {|f, v| v } }
-        assert_equal(nil, header)
-        assert_equal([[1, 3]], all)
-      }
-      Tb.open_reader(ic) {|r|
-        header = nil
-        all = []
-        r.with_header {|h| header = h}.each {|pairs| all << pairs.map {|f, v| v } }
-        assert_equal(%w[a b], header)
-        assert_equal([%w[1 3]], all)
-      }
-      assert_raise(ArgumentError) { Tb.open_reader(Object.new) }
     }
   end
 
-  def test_field_index
+  def test_open_prefix_json
     Dir.mktmpdir {|d|
-      open(i="#{d}/i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
-        a,b
-        1,3
+      open(ij="#{d}/j", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
+        [{"a":1,"b":3}]
       End
-      Tb.open_reader(i) {|r|
+      Tb.open_reader("json:#{ij}") {|r|
         header = nil
         all = []
         r.with_header {|h| header = h}.each {|pairs| all << pairs.map {|f, v| v } }
         assert_equal(%w[a b], header)
-        assert_equal([%w[1 3]], all)
-        assert_equal("a", r.field_from_index(0))
-        assert_equal("b", r.field_from_index(1))
-        assert_raise(ArgumentError) { r.field_from_index(2) }
-        assert_equal(0, r.index_from_field("a"))
-        assert_equal(1, r.index_from_field("b"))
-        assert_raise(ArgumentError) { r.index_from_field("c") }
-        assert_raise(ArgumentError) { r.index_from_field("2") }
-        assert_equal("1", r.field_from_index_ex(2))
-        assert_equal(3, r.index_from_field_ex("2"))
+        assert_equal([[1, 3]], all)
       }
     }
   end
 
-  def test_header_ignore_empty
-    csv = "\n" + <<-'End'.gsub(/^[ \t]+/, '')
-    a,b
-    1,2
-    3,4
-    End
-    t = Tb.parse_csv(csv)
-    records = []
-    t.each_record {|record|
-      records << record.to_h
-    }
-    assert_equal(
-      [{"a"=>"1", "b"=>"2"},
-       {"a"=>"3", "b"=>"4"}],
-      records)
+  def test_open_nonstring
+    assert_raise(ArgumentError) { Tb.open_reader(Object.new) }
   end
-
-  def test_header_empty_only
-    csv = "\n"
-    t = Tb.parse_csv(csv)
-    records = []
-    t.each_record {|record|
-      records << record.to_h
-    }
-    assert_equal([], records)
-  end
-
 end

data/test/test_tbenum.rb

@@ -4,14 +4,14 @@ require 'tmpdir'
 
 class TestTbEnum < Test::Unit::TestCase
   def test_cat_without_block
-    t1 = Tb.new(%w[a b], [1, 2], [3, 4])
-    t2 = Tb.new(%w[c d], [5, 6], [7, 8])
+    t1 = Tb::Enumerator.from_header_and_values(%w[a b], [1, 2], [3, 4])
+    t2 = Tb::Enumerator.from_header_and_values(%w[c d], [5, 6], [7, 8])
     e = t1.cat(t2)
     result = []
     e.with_header {|header|
       result << [:header, header]
     }.each {|x|
-      assert_kind_of(Tb::Record, x)
+      assert_kind_of(Hash, x)
       result << [x.to_a]
     }
     assert_equal(
@@ -24,11 +24,11 @@ class TestTbEnum < Test::Unit::TestCase
   end
 
   def test_cat_with_block
-    t1 = Tb.new(%w[a b], [1, 2], [3, 4])
-    t2 = Tb.new(%w[c d], [5, 6], [7, 8])
+    t1 = Tb::Enumerator.from_header_and_values(%w[a b], [1, 2], [3, 4])
+    t2 = Tb::Enumerator.from_header_and_values(%w[c d], [5, 6], [7, 8])
     result = []
     t1.cat(t2) {|x|
-      assert_kind_of(Tb::Record, x)
+      assert_kind_of(Hash, x)
       result << x.to_a
     }
     assert_equal(
@@ -41,9 +41,9 @@ class TestTbEnum < Test::Unit::TestCase
 
   def test_write_to_csv_basic
     obj = [
-      [['a', 0], ['b', 1], ['c', 2]],
-      [['a', 3], ['b', 4], ['c', 5]],
-      [['a', 6], ['b', 7], ['c', 8]],
+      {'a' => 0, 'b' => 1, 'c' => 2},
+      {'a' => 3, 'b' => 4, 'c' => 5},
+      {'a' => 6, 'b' => 7, 'c' => 8},
     ]
     def obj.header_and_each(header_proc, &block)
       header_proc.call(nil) if header_proc
@@ -65,9 +65,9 @@ class TestTbEnum < Test::Unit::TestCase
 
   def test_write_to_csv_header_extension
     obj = [
-      [['a', 0]],
-      [['b', 1]],
-      [['c', 2]],
+      {'a' => 0},
+      {'b' => 1},
+      {'c' => 2},
     ]
     def obj.header_and_each(header_proc, &block)
       header_proc.call(nil) if header_proc
@@ -89,9 +89,9 @@ class TestTbEnum < Test::Unit::TestCase
 
   def test_write_to_csv_without_header
     obj = [
-      [['a', 0]],
-      [['b', 1]],
-      [['c', 2]],
+      {'1' => "x"},
+      {'2' => "y"},
+      {'3' => "z"},
     ]
     def obj.header_and_each(header_proc, &block)
       header_proc.call(nil) if header_proc
@@ -103,9 +103,9 @@ class TestTbEnum < Test::Unit::TestCase
         obj.write_to_csv(f, false)
       }
       assert_equal(<<-'End'.gsub(/^\s*/, ''), File.read("#{d}/foo.csv"))
-        0
-        ,1
-        ,,2
+        x
+        ,y
+        ,,z
       End
     }
   end

data/test/test_tsv.rb

@@ -2,47 +2,36 @@ require 'tb'
 require 'test/unit'
 
 class TestTbTSV < Test::Unit::TestCase
+  def parse_tsv(tsv)
+    Tb::HeaderTSVReader.new(StringIO.new(tsv)).to_a
+  end
+
+  def generate_tsv(ary)
+    writer = Tb::HeaderTSVWriter.new(out = '')
+    ary.each {|h| writer.put_hash h }
+    writer.finish
+    out
+  end
+
   def test_parse
     tsv = "a\tb\n1\t2\n"
-    t = Tb.parse_tsv(tsv)
-    records = []
-    t.each_record {|record|
-      records << record.to_h_with_reserved
-    }
+    ary = parse_tsv(tsv)
     assert_equal(
-      [{"_recordid"=>0, "a"=>"1", "b"=>"2"}],
-      records)
+      [{"a"=>"1", "b"=>"2"}],
+      ary)
   end
 
-  def test_parse_conv
-    tsv = "foo\na\tb\n1\t2\n"
-    t = Tb.parse_tsv(tsv) {|aa|
-      assert_equal([%w[foo],
-                   %w[a b],
-                   %w[1 2]],
-                   aa)
-      aa.shift
-      aa
-    }
-    records = []
-    t.each_record {|record|
-      records << record.to_h_with_reserved
-    }
+  def test_parse2
+    tsv = "a\tb\n" + "1\t2\n" + "3\t4\n"
+    ary = parse_tsv(tsv)
     assert_equal(
-      [{"_recordid"=>0, "a"=>"1", "b"=>"2"}],
-      records)
+      [{"a"=>"1", "b"=>"2"}, {"a"=>"3", "b"=>"4"}],
+      ary)
   end
 
   def test_generate_tsv
-    tbl = Tb.new %w[a b], %w[foo bar]
-    tbl.generate_tsv(out="")
-    assert_equal("a\tb\nfoo\tbar\n", out)
-  end
-
-  def test_generate_tsv_with_block
-    tbl = Tb.new %w[a b], %w[foo bar], %w[q w]
-    tbl.generate_tsv(out="") {|recids| recids.reverse }
-    assert_equal("a\tb\nq\tw\nfoo\tbar\n", out)
+    t = [{'a' => 'foo', 'b' => 'bar'}]
+    assert_equal("a\tb\nfoo\tbar\n", generate_tsv(t))
   end
 
 end

data/test/util_tbtest.rb

@@ -0,0 +1,12 @@
+require 'stringio'
+
+def capture_stderr
+  begin
+    save_stderr = $stderr
+    $stderr = StringIO.new(stderr='')
+    yield
+  ensure
+    $stderr = save_stderr
+  end
+  stderr
+end

metadata

@@ -1,24 +1,40 @@
 --- !ruby/object:Gem::Specification
 name: tb
 version: !ruby/object:Gem::Version
-  version: '0.9'
+  version: '1.0'
 platform: ruby
 authors:
 - Tanaka Akira
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-04-29 00:00:00.000000000 Z
-dependencies: []
+date: 2014-10-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.5.5
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.5.5
 description: |
-  tb is a manipulation tool for table: CSV, TSV, LTSV, JSON, etc.
+  tb is a manipulation tool for tables.
 
   tb provides a command and a library for manipulating tables:
-  Unix filter like operations (grep, sort, cat, cut, ls, etc.),
+  Unix filter like operations (sort, cat, cut, ls, etc.),
   SQL like operations (join, group, etc.),
-  other table operations (gsub, rename, cross, melt, unmelt, etc.),
+  other table operations (search, gsub, rename, cross, melt, unmelt, etc.),
   information extractions (git, svn, tar),
   and more.
+
+  tb supports various table formats: CSV, TSV, JSON, NDJSON, LTSV, etc.
 email: akr@fsij.org
 executables:
 - tb
@@ -28,7 +44,6 @@ files:
 - README
 - bin/tb
 - lib/tb.rb
-- lib/tb/basic.rb
 - lib/tb/catreader.rb
 - lib/tb/cmd_cat.rb
 - lib/tb/cmd_consecutive.rb
@@ -36,7 +51,6 @@ files:
 - lib/tb/cmd_cross.rb
 - lib/tb/cmd_cut.rb
 - lib/tb/cmd_git.rb
-- lib/tb/cmd_grep.rb
 - lib/tb/cmd_group.rb
 - lib/tb/cmd_gsub.rb
 - lib/tb/cmd_help.rb
@@ -47,6 +61,7 @@ files:
 - lib/tb/cmd_nest.rb
 - lib/tb/cmd_newfield.rb
 - lib/tb/cmd_rename.rb
+- lib/tb/cmd_search.rb
 - lib/tb/cmd_shape.rb
 - lib/tb/cmd_sort.rb
 - lib/tb/cmd_svn.rb
@@ -70,15 +85,19 @@ files:
 - lib/tb/enumerator.rb
 - lib/tb/ex_enumerable.rb
 - lib/tb/ex_enumerator.rb
-- lib/tb/fieldset.rb
 - lib/tb/fileenumerator.rb
 - lib/tb/func.rb
+- lib/tb/hashreader.rb
+- lib/tb/hashwriter.rb
+- lib/tb/headerreader.rb
+- lib/tb/headerwriter.rb
 - lib/tb/json.rb
 - lib/tb/ltsv.rb
+- lib/tb/ndjson.rb
+- lib/tb/numericreader.rb
+- lib/tb/numericwriter.rb
 - lib/tb/pager.rb
 - lib/tb/pnm.rb
-- lib/tb/reader.rb
-- lib/tb/record.rb
 - lib/tb/revcmp.rb
 - lib/tb/ropen.rb
 - lib/tb/search.rb
@@ -91,9 +110,9 @@ files:
 - sample/poi-xls2csv.rb
 - sample/poi-xls2csv.sh
 - sample/tbplot
+- tb.gemspec
 - test-all-cov.rb
 - test-all.rb
-- test/test_basic.rb
 - test/test_catreader.rb
 - test/test_cmd_cat.rb
 - test/test_cmd_consecutive.rb
@@ -131,19 +150,21 @@ files:
 - test/test_customcmp.rb
 - test/test_customeq.rb
 - test/test_ex_enumerable.rb
-- test/test_fieldset.rb
 - test/test_fileenumerator.rb
+- test/test_headercsv.rb
 - test/test_json.rb
 - test/test_ltsv.rb
+- test/test_ndjson.rb
+- test/test_numericcsv.rb
 - test/test_pager.rb
 - test/test_pnm.rb
 - test/test_reader.rb
-- test/test_record.rb
 - test/test_revcmp.rb
 - test/test_search.rb
 - test/test_tbenum.rb
 - test/test_tsv.rb
 - test/test_zipper.rb
+- test/util_tbtest.rb
 homepage: https://github.com/akr/tb
 licenses: []
 metadata: {}
@@ -163,12 +184,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.2
 signing_key: 
 specification_version: 4
-summary: 'manipulation tool for tables: CSV, TSV, LTSV, JSON, etc.'
+summary: 'manipulation tool for tables: CSV, TSV, LTSV, JSON, NDJSON, etc.'
 test_files:
-- test/test_basic.rb
 - test/test_catreader.rb
 - test/test_cmd_cat.rb
 - test/test_cmd_consecutive.rb
@@ -206,16 +226,18 @@ test_files:
 - test/test_customcmp.rb
 - test/test_customeq.rb
 - test/test_ex_enumerable.rb
-- test/test_fieldset.rb
 - test/test_fileenumerator.rb
+- test/test_headercsv.rb
 - test/test_json.rb
 - test/test_ltsv.rb
+- test/test_ndjson.rb
+- test/test_numericcsv.rb
 - test/test_pager.rb
 - test/test_pnm.rb
 - test/test_reader.rb
-- test/test_record.rb
 - test/test_revcmp.rb
 - test/test_search.rb
 - test/test_tbenum.rb
 - test/test_tsv.rb
 - test/test_zipper.rb
+- test/util_tbtest.rb