tb 0.9 → 1.0

data/test/test_cmd_shape.rb

@@ -15,7 +15,44 @@ class TestTbCmdShape < Test::Unit::TestCase
     FileUtils.rmtree @tmpdir
   end
 
-  def test_basic
+  def test_ndjson
+    File.open(i="i.ndjson", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
+      {"a":0, "b":1}
+      {"a":4, "b":5, "c":6}
+    End
+    Tb::Cmd.main_shape(['-o', o="o.csv", i])
+    assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
+      filename,records,min_pairs,max_pairs
+      i.ndjson,2,2,3
+    End
+  end
+
+  def test_csv
+    File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
+      a,b,c
+      0,1
+      4,5,6
+    End
+    Tb::Cmd.main_shape(['-o', o="o.csv", i])
+    assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
+      filename,records,min_pairs,max_pairs,header_fields,min_fields,max_fields
+      i.csv,2,2,3,3,2,3
+    End
+  end
+
+  def test_output_ndjson
+    File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
+      a,b,c
+      0,1
+      4,5,6
+    End
+    Tb::Cmd.main_shape(['-o', o="o.ndjson", i])
+    assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
+      {"filename":"i.csv","records":2,"min_pairs":2,"max_pairs":3,"header_fields":3,"min_fields":2,"max_fields":3}
+    End
+  end
+
+  def test_extra_field
     File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
       a,b,c
       0,1
@@ -23,8 +60,8 @@ class TestTbCmdShape < Test::Unit::TestCase
     End
     Tb::Cmd.main_shape(['-o', o="o.csv", i])
     assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
-      header_fields,min_fields,max_fields,records,filename
-      3,2,4,2,i.csv
+      filename,records,min_pairs,max_pairs,header_fields,min_fields,max_fields
+      i.csv,2,2,3,3,2,4
     End
   end
 
@@ -41,9 +78,9 @@ class TestTbCmdShape < Test::Unit::TestCase
     End
     Tb::Cmd.main_shape(['-o', o="o.csv", i1, i2])
     assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
-      header_fields,min_fields,max_fields,records,filename
-      1,1,1,2,i1.csv
-      2,2,2,2,i2.csv
+      filename,records,min_pairs,max_pairs,header_fields,min_fields,max_fields
+      i1.csv,2,1,1,1,1,1
+      i2.csv,2,2,2,2,2,2
     End
   end
 

data/test/test_cmd_svn_log.rb

@@ -34,10 +34,10 @@ class TestTbCmdSvnLog < Test::Unit::TestCase
     system("svn commit -q -m baz foo hoge")
     system("svn update -q") # update the revision of the directory.
     Tb::Cmd.main_svn(['-o', o="o.csv"])
-    result = File.read(o)
-    tb = Tb.parse_csv(result)
-    assert_equal(1, tb.size)
-    assert_match(/baz/, tb.get_record(0)["msg"])
+    aa = CSV.read(o)
+    assert_equal(2, aa.length)
+    header, row = aa
+    assert_match(/baz/, row[header.index "msg"])
   end
 
   def test_verbose
@@ -49,16 +49,15 @@ class TestTbCmdSvnLog < Test::Unit::TestCase
     system("svn commit -q -m baz foo hoge")
     system("svn update -q") # update the revision of the directory.
     Tb::Cmd.main_svn(['-o', o="o.csv", '--', '-v'])
-    result = File.read(o)
-    tb = Tb.parse_csv(result)
-    assert_equal(2, tb.size)
-    recs = [tb.get_record(0), tb.get_record(1)]
-    recs = recs.sort_by {|rec| rec['path'] }
-    recs.each {|rec|
-      assert_match(/baz/, rec["msg"])
+    aa = CSV.read(o)
+    assert_equal(3, aa.length)
+    header, *rows = aa
+    rows = rows.sort_by {|rec| rows[header.index 'path'] }
+    rows.each {|row|
+      assert_match(/baz/, row[header.index "msg"])
     }
-    assert_match(/foo/, recs[0]["path"])
-    assert_match(/hoge/, recs[1]["path"])
+    assert_match(/foo/, rows[0][header.index "path"])
+    assert_match(/hoge/, rows[1][header.index "path"])
   end
 
   def test_log_xml
@@ -71,18 +70,18 @@ class TestTbCmdSvnLog < Test::Unit::TestCase
     system("svn update -q") # update the revision of the directory.
     ###
     Tb::Cmd.main_svn(['-o', o="o.csv"])
-    result = File.read(o)
-    tb = Tb.parse_csv(result)
-    assert_equal(1, tb.size)
-    assert_match(/baz/, tb.get_record(0)["msg"])
+    aa = CSV.read(o)
+    assert_equal(2, aa.length)
+    header, row = aa
+    assert_match(/baz/, row[header.index "msg"])
     ###
     system("svn log --xml > log.xml")
     FileUtils.rmtree('.svn')
     Tb::Cmd.main_svn(['-o', o="o.csv", '--svn-log-xml=log.xml'])
-    result = File.read(o)
-    tb = Tb.parse_csv(result)
-    assert_equal(1, tb.size)
-    assert_match(/baz/, tb.get_record(0)["msg"])
+    aa = CSV.read(o)
+    assert_equal(2, aa.length)
+    header, row = aa
+    assert_match(/baz/, row[header.index "msg"])
   end
 
   def test_no_props
@@ -98,11 +97,11 @@ class TestTbCmdSvnLog < Test::Unit::TestCase
     system("svn propdel -q svn:log --revprop -r 1 .")
     ###
     Tb::Cmd.main_svn(['-o', o="o.csv"])
-    result = File.read(o)
-    tb = Tb.parse_csv(result)
-    assert_equal(1, tb.size)
-    assert_equal('(no author)', tb.get_record(0)["author"])
-    assert_equal('(no date)', tb.get_record(0)["date"])
-    assert_equal('', tb.get_record(0)["msg"])
+    aa = CSV.read(o)
+    assert_equal(2, aa.length)
+    header, row = aa
+    assert_equal('(no author)', row[header.index "author"])
+    assert_equal('(no date)', row[header.index "date"])
+    assert_equal('', row[header.index "msg"])
   end
 end

data/test/test_cmd_to_csv.rb

@@ -1,6 +1,7 @@
 require 'test/unit'
 require 'tb/cmdtop'
 require 'tmpdir'
+require_relative 'util_tbtest'
 
 class TestTbCmdToCSV < Test::Unit::TestCase
   def setup
@@ -49,18 +50,22 @@ class TestTbCmdToCSV < Test::Unit::TestCase
     End
   end
 
-  def test_complement_header
+  def test_short_header
     File.open(i="i.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
       a,b
       0,1,2
       4,5,6
     End
-    Tb::Cmd.main_to_csv(['-o', o="o.csv", i])
+    o = "o.csv"
+    stderr = capture_stderr {
+      Tb::Cmd.main_to_csv(['-o', o, i])
+    }
     assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
-      a,b,1
-      0,1,2
-      4,5,6
+      a,b
+      0,1
+      4,5
     End
+    assert_match(/Header too short/, stderr)
   end
 
   def test_numeric

data/test/test_cmd_to_json.rb

@@ -68,6 +68,22 @@ class TestTbCmdToJSON < Test::Unit::TestCase
     End
   end
 
+  def test_ltsv_to_json1
+    File.open(i="i.ltsv", "w") {|f| f << "foo:bar\n" }
+    Tb::Cmd.main_to_json(['-o', o="o.json", i])
+    assert_equal(<<-"End".gsub(/\s/, ''), File.read(o).gsub(/\s/, ''))
+      [{"foo":"bar"}]
+    End
+  end
+
+  def test_ltsv_to_json2
+    File.open(i="i.ltsv", "w") {|f| f << "foo:bar\nbaz:qux\n" }
+    Tb::Cmd.main_to_json(['-o', o="o.json", i])
+    assert_equal(<<-"End".gsub(/\s/, ''), File.read(o).gsub(/\s/, ''))
+      [{"foo":"bar"}, {"baz":"qux"}]
+    End
+  end
+
   def test_twofile
     File.open(i1="i1.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
       a,b

data/test/test_cmd_to_ltsv.rb

@@ -57,8 +57,8 @@ class TestTbCmdToLTSV < Test::Unit::TestCase
     assert_equal(<<-"End".gsub(/^[ \t]+/, ''), File.read(o))
       a:1\tb:2
       a:3\tb:4
-      a:6\tb:5
-      a:8\tb:7
+      b:5\ta:6
+      b:7\ta:8
     End
   end
 

data/test/test_cmd_to_pp.rb

@@ -1,6 +1,7 @@
 require 'test/unit'
 require 'tb/cmdtop'
 require 'tmpdir'
+require_relative 'util_tbtest'
 
 class TestTbCmdToPP < Test::Unit::TestCase
   def setup
@@ -33,10 +34,14 @@ class TestTbCmdToPP < Test::Unit::TestCase
       a,b
       0,1,2,3
     End
-    Tb::Cmd.main_to_pp(['-o', o="o.pp", i])
+    o = "o.pp"
+    stderr = capture_stderr {
+      Tb::Cmd.main_to_pp(['-o', o, i])
+    }
     assert_equal(<<-"End".gsub(/\s/, ''), File.read(o).gsub(/\s/, ''))
-      { "a" => "0", "b" => "1", "1" => "2", "2" => "3" }
+      { "a" => "0", "b" => "1" }
     End
+    assert_match(/Header too short/, stderr)
   end
 
   def test_twofile

data/test/test_csv.rb

@@ -1,98 +1,110 @@
 require 'tb'
 require 'test/unit'
+require_relative 'util_tbtest'
 
 class TestTbCSV < Test::Unit::TestCase
+  def parse_csv(csv)
+    Tb::HeaderCSVReader.new(StringIO.new(csv)).to_a
+  end
+
+  def generate_csv(ary)
+    writer = Tb::HeaderCSVWriter.new(out = '')
+    ary.each {|h| writer.put_hash h }
+    writer.finish
+    out
+  end
+
   def test_parse
-    csv = <<-'End'.gsub(/^\s+/, '')
+    t = parse_csv(<<-'End'.gsub(/^\s+/, ''))
+    a,b
+    1,2
+    3,4
+    End
+    assert_equal(
+      [{"a"=>"1", "b"=>"2"},
+       {"a"=>"3", "b"=>"4"}],
+      t)
+  end
+
+  def test_parse_empty_line_before_header
+    empty_line = "\n"
+    t = parse_csv(empty_line + <<-'End'.gsub(/^\s+/, ''))
     a,b
     1,2
     3,4
     End
-    t = Tb.parse_csv(csv)
-    records = []
-    t.each_record {|record|
-      records << record.to_h_with_reserved
-    }
     assert_equal(
-      [{"_recordid"=>0, "a"=>"1", "b"=>"2"},
-       {"_recordid"=>1, "a"=>"3", "b"=>"4"}],
-      records)
+      [{"a"=>"1", "b"=>"2"},
+       {"a"=>"3", "b"=>"4"}],
+      t)
   end
 
-  def test_parse_empty
-    csv = <<-'End'.gsub(/^\s+/, '')
+  def test_parse_empty_value
+    t = parse_csv(<<-'End'.gsub(/^\s+/, ''))
     a,b,c
     1,,2
     3,"",4
     End
-    t = Tb.parse_csv(csv)
-    records = []
-    t.each_record {|record|
-      records << record.to_h_with_reserved
-    }
     assert_equal(
-      [{"_recordid"=>0, "a"=>"1", "c"=>"2"},
-       {"_recordid"=>1, "a"=>"3", "b"=>"", "c"=>"4"}],
-      records)
+      [{"a"=>"1", "b"=>nil, "c"=>"2"},
+       {"a"=>"3", "b"=>"", "c"=>"4"}],
+      t)
   end
 
-  def test_parse_conv
-    csv = "foo\na,b\n1,2\n"
-    t = Tb.parse_csv(csv) {|aa|
-      assert_equal([%w[foo],
-                   %w[a b],
-                   %w[1 2]],
-                   aa)
-      aa.shift
-      aa
-    }
-    records = []
-    t.each_record {|record|
-      records << record.to_h_with_reserved
-    }
-    assert_equal(
-      [{"_recordid"=>0, "a"=>"1", "b"=>"2"}],
-      records)
+  def test_parse_newline
+    t = parse_csv("\n")
+    assert_equal([], t)
   end
 
   def test_generate
-    t = Tb.new %w[a b], [1, 2], [3, 4]
-    out = t.generate_csv('', ['a', 'b'])
-    assert_equal(<<-'End'.gsub(/^\s+/, ''), out)
+    t = [{'a' => 1, 'b' => 2},
+         {'a' => 3, 'b' => 4}]
+    assert_equal(<<-'End'.gsub(/^\s+/, ''), generate_csv(t))
     a,b
     1,2
     3,4
     End
   end
 
-  def test_generate_without_explicit_fields
-    t = Tb.new %w[a b], [1, 2], [3, 4]
-    out = t.generate_csv('')
-    assert_equal(<<-'End'.gsub(/^\s+/, ''), out)
-    a,b
-    1,2
-    3,4
-    End
-  end
-
-  def test_generate_with_block
-    t = Tb.new %w[a b], [1, 2], [3, 4]
-    out = t.generate_csv('', ['a', 'b']) {|recids| recids.reverse }
-    assert_equal(<<-'End'.gsub(/^\s+/, ''), out)
-    a,b
-    3,4
-    1,2
-    End
-  end
-
   def test_generate_empty
-    t = Tb.new %w[a b c], [1, nil, 2], [3, '', 4]
-    out = t.generate_csv('', ['a', 'b', 'c'])
-    assert_equal(<<-'End'.gsub(/^\s+/, ''), out)
+    t = [{'a' => 1, 'b' => nil, 'c' => 2},
+         {'a' => 3, 'b' => '',  'c' => 4}]
+    assert_equal(<<-'End'.gsub(/^\s+/, ''), generate_csv(t))
     a,b,c
     1,,2
     3,"",4
     End
   end
 
+  def test_parse_ambiguous_header
+    t = nil
+    stderr = capture_stderr {
+      t = parse_csv(<<-'End'.gsub(/^\s+/, ''))
+      a,b,a,b,c
+      0,1,2,3,4
+      5,6,7,8,9
+      End
+    }
+    assert_equal(
+      [{"a"=>"0", "b"=>"1", "c"=>"4"},
+       {"a"=>"5", "b"=>"6", "c"=>"9"}],
+      t)
+    assert_match(/Ambiguous header field/, stderr)
+  end
+
+  def test_parse_empty_header_field
+    t = nil
+    stderr = capture_stderr {
+      t = parse_csv(<<-'End'.gsub(/^\s+/, ''))
+      a,,c
+      0,1,2
+      5,6,7
+      End
+    }
+    assert_equal(
+      [{"a"=>"0", "c"=>"2"},
+       {"a"=>"5", "c"=>"7"}],
+      t)
+    assert_match(/Empty header field/, stderr)
+  end
 end

data/test/test_ex_enumerable.rb

@@ -46,7 +46,6 @@ class TestTbEnumerable < Test::Unit::TestCase
                  a.tb_categorize(:color, lambda {|e| true }, :seed=>0, :op=>lambda {|s, v| s+1 }))
 
     assert_raise(ArgumentError) { a.tb_categorize(:color, lambda {|e| true }, :seed=>0,
-                                               :seed=>nil,
                                                :op=>lambda {|s, v| s+1 },
                                                :update=>lambda {|ks, s, v| s+1 }) }
 

data/test/test_fileenumerator.rb

@@ -204,7 +204,7 @@ class TestTbFileEnumerator < Test::Unit::TestCase
   end
 
   def test_to_fileheaderenumerator_reader
-    tb = Tb.new %w[a b], [1, 2], [3, 4]
+    tb = Tb::Enumerator.from_header_and_values %w[a b], [1, 2], [3, 4]
     fe = tb.to_fileenumerator
     iter = fe.each
     assert_respond_to(iter, :next)
@@ -222,7 +222,7 @@ class TestTbFileEnumerator < Test::Unit::TestCase
   end
 
   def test_to_fileheaderenumerator_with_header_reader
-    tb = Tb.new %w[a b], [1, 2], [3, 4]
+    tb = Tb::Enumerator.from_header_and_values %w[a b], [1, 2], [3, 4]
     header = nil
     fe = tb.with_header {|h0|
       header = h0
@@ -244,7 +244,7 @@ class TestTbFileEnumerator < Test::Unit::TestCase
   end
 
   def test_fileheaderenumerator_open_reader
-    tb = Tb.new %w[a b], [1, 2], [3, 4]
+    tb = Tb::Enumerator.from_header_and_values %w[a b], [1, 2], [3, 4]
     fe = tb.to_fileenumerator
     iter0 = nil
     fe.open_reader {|iter|