tb 0.3 → 0.4

data/lib/tb/ex_enumerator.rb

@@ -0,0 +1,102 @@
+# Copyright (C) 2012 Tanaka Akira  <akr@fsij.org>
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 
+#  1. Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#  2. Redistributions in binary form must reproduce the above
+#     copyright notice, this list of conditions and the following
+#     disclaimer in the documentation and/or other materials provided
+#     with the distribution.
+#  3. The name of the author may not be used to endorse or promote
+#     products derived from this software without specific prior
+#     written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+# GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+module Tb::ExEnumerator
+  # :call-seq:
+  #
+  #   Tb::ExEnumerator.merge_sorted(enumerator1, ...) {|key, enumerator1_or_nil, ...| ... }
+  #
+  # iterates over enumerators specified by arguments.
+  #
+  # The enumerators should yield an array which first element is a comparable key.
+  # The enumerators should be sorted by the key in ascending order.
+  #
+  # Tb::Enumerator.merge_sorted iterates keys in all of the enumerators.
+  # It yields an array which contains a key and enumerators which has the key.
+  # The array contains nil if corresponding enumerator don't have the key. 
+  #
+  # The block may or may not use +next+ method to advance enumerators.
+  # Anyway Tb::Enumerator.merge_sorted advance enumerators until peeked key is
+  # greater than the yielded key.
+  #
+  # If a enumerator has multiple elements with same key,
+  # the block can read them using +next+ method.
+  # +peek+ method should also be used to determine the key of the next element has
+  # the current key or not.
+  # Be careful to not consume extra elements with different key.
+  #
+  def (Tb::ExEnumerator).merge_sorted(*enumerators) # :yields: [key, enumerator1_or_nil, ...]
+    while true
+      has_min = false
+      min = nil
+      min_enumerators = []
+      enumerators.each {|kpe|
+        begin
+          key, = kpe.peek
+        rescue StopIteration
+          min_enumerators << nil
+          next
+        end
+        if !has_min
+          has_min = true
+          min = key
+          min_enumerators << kpe
+        else
+          cmp = key <=> min
+          if cmp < 0
+            min_enumerators.fill(nil)
+            min_enumerators << kpe
+            min = key
+          elsif cmp == 0
+            min_enumerators << kpe
+          else
+            min_enumerators << nil
+          end
+        end
+      }
+      if !has_min
+        return
+      end
+      yield [min, *min_enumerators]
+      min_enumerators.each {|kpe|
+        next if !kpe
+        while true
+          begin
+            key, = kpe.peek
+          rescue StopIteration
+            break
+          end
+          if (min <=> key) < 0
+            break
+          end
+          kpe.next
+        end
+      }
+    end
+  end
+end

data/lib/tb/fileenumerator.rb

@@ -282,7 +282,7 @@ class Tb::FileEnumerator
   end
 end
 
-module Tb::Enum
+module Tb::Enumerable
   # creates a Tb::FileHeaderEnumerator object.
   #
   def to_fileenumerator
@@ -299,7 +299,7 @@ module Tb::Enum
 end
 
 class Tb::FileHeaderEnumerator < Tb::FileEnumerator
-  include Tb::Enum
+  include Tb::Enumerable
 
   class HBuilder
     def initialize(klass)

data/lib/tb/func.rb

@@ -0,0 +1,141 @@
+# Copyright (C) 2012 Tanaka Akira  <akr@fsij.org>
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 
+#  1. Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#  2. Redistributions in binary form must reproduce the above
+#     copyright notice, this list of conditions and the following
+#     disclaimer in the documentation and/or other materials provided
+#     with the distribution.
+#  3. The name of the author may not be used to endorse or promote
+#     products derived from this software without specific prior
+#     written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+# GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+module Tb::Func
+  def self.smart_cmp_value(v)
+    case v
+    when nil
+      []
+    when Numeric
+      [0, v]
+    when String
+      if v.respond_to? :force_encoding
+        v = v.dup.force_encoding("ASCII-8BIT")
+      end
+      case v
+      when /\A\s*-?\d+\s*\z/
+        [0, v.to_i(10)]
+      when /\A\s*-?(\d+(\.\d+)?)([eE][-+]?\d+)?\s*\z/
+        [0, Float(v)]
+      else
+        a = []
+        v.scan(/(\d+)|\D+/) {
+          if $1
+            a << 0 << $1.to_i
+          else
+            a << 1 << $&
+          end
+        }
+        a
+      end
+    else
+      raise ArgumentError, "unexpected: #{v.inspect}"
+    end
+  end
+
+  def self.smart_numerize(v)
+    return v if v.kind_of? Numeric
+    v = v.strip
+    if /\A-?\d+\z/ =~ v
+      v = v.to_i
+    elsif /\A-?(\d+(\.\d*)?|\.\d+)([eE][-+]?\d+)?\z/ =~ v
+      v = v.to_f
+    else
+      raise ArgumentError, "number string expected: #{v.inspect}"
+    end
+    v
+  end
+
+  module Count; end
+  def Count.start(value) 1 end
+  def Count.call(v1, v2) v1 + v2 end
+  def Count.aggregate(count) count end
+
+  module Sum; end
+  def Sum.start(value) Tb::Func.smart_numerize(value) end
+  def Sum.call(v1, v2) v1 + v2 end
+  def Sum.aggregate(sum) sum end
+
+  module Min; end
+  def Min.start(value) [value, Tb::Func.smart_cmp_value(value)]  end
+  def Min.call(vc1, vc2) (vc1.last <=> vc2.last) <= 0 ? vc1 : vc2 end
+  def Min.aggregate(vc) vc.first end
+
+  module Max; end
+  def Max.start(value) [value, Tb::Func.smart_cmp_value(value)]  end
+  def Max.call(vc1, vc2) (vc1.last <=> vc2.last) >= 0 ? vc1 : vc2 end
+  def Max.aggregate(vc) vc.first end
+
+  module Avg; end
+  def Avg.start(value) [Tb::Func.smart_numerize(value), 1] end
+  def Avg.call(v1, v2) [v1[0] + v2[0], v1[1] + v2[1]] end
+  def Avg.aggregate(sum_count) sum_count[0] / sum_count[1].to_f end
+
+  module First; end
+  def First.start(value) value end
+  def First.call(v1, v2) v1 end
+  def First.aggregate(value) value end
+
+  module Last; end
+  def Last.start(value) value end
+  def Last.call(v1, v2) v2 end
+  def Last.aggregate(value) value end
+
+  module Values; end
+  def Values.start(value) [value] end
+  def Values.call(a1, a2) a1.concat a2 end
+  def Values.aggregate(ary) ary.join(',') end
+
+  module UniqueValues; end
+  def UniqueValues.start(value) {value => true} end
+  def UniqueValues.call(h1, h2) h1.update h2 end
+  def UniqueValues.aggregate(hash) hash.keys.join(',') end
+
+  class FirstN
+    def initialize(n) @n = n end
+    def start(value) [value] end
+    def call(a1, a2) a1.length == @n ? a1 : (a1+a2).first(@n) end
+    def aggregate(ary) ary end
+  end
+
+  class LastN
+    def initialize(n) @n = n end
+    def start(value) [value] end
+    def call(a1, a2) a2.length == @n ? a2 : (a1+a2).last(@n) end
+    def aggregate(ary) ary end
+  end
+
+  AggregationFunctions = {}
+  Tb::Func.constants.each {|c|
+    v = Tb::Func.const_get(c)
+    if v.respond_to? :aggregate
+      AggregationFunctions[c.to_s.downcase] = v
+    end
+  }
+
+end

data/lib/tb/json.rb

@@ -30,7 +30,7 @@ require 'json'
 
 class Tb
   class JSONReader
-    include Tb::Enum
+    include Tb::Enumerable
 
     def initialize(string)
       @ary = JSON.parse(string)

data/lib/tb/reader.rb

@@ -29,7 +29,7 @@
 # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 class Tb::Reader
-  include Tb::Enum
+  include Tb::Enumerable
 
   def initialize(opts={}, &rawreader_open)
     @opt_n = opts[:numeric]
@@ -89,12 +89,12 @@ class Tb::Reader
       h = self.internal_header(rawreader)
       header_proc.call(h) if header_proc
       while ary = self.internal_shift(rawreader)
-        pairs = []
+        pairs = {}
         ary.each_with_index {|v, i|
           f = field_from_index_ex(i)
-          pairs << [f, v]
+          pairs[f] = v
         }
-        yield Tb::Pairs.new(pairs)
+        yield pairs
       end
     }
     @reader_open.call(body)

data/lib/tb/search.rb

@@ -441,8 +441,7 @@ module Tb::Search::EmptyState
     elsif !rest.empty?
       return rest[0]
     else
-      exc = defined?(KeyError) ? KeyError : IndexError # 1.9 v.s. 1.8
-      raise exc, "key not found: #{k}"
+      raise KeyError, "key not found: #{k}"
     end
   end
 
@@ -526,8 +525,7 @@ class Tb::Search::State
     elsif !rest.empty?
       return rest[0]
     else
-      exc = defined?(KeyError) ? KeyError : IndexError # 1.9 v.s. 1.8
-      raise exc, "key not found: #{k}"
+      raise KeyError, "key not found: #{k}"
     end
   end
 

data/lib/tb/zipper.rb

@@ -0,0 +1,60 @@
+# Copyright (C) 2012 Tanaka Akira  <akr@fsij.org>
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 
+#  1. Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#  2. Redistributions in binary form must reproduce the above
+#     copyright notice, this list of conditions and the following
+#     disclaimer in the documentation and/or other materials provided
+#     with the distribution.
+#  3. The name of the author may not be used to endorse or promote
+#     products derived from this software without specific prior
+#     written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
+# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+# GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+class Tb::Zipper
+  def initialize(ops)
+    @ops = ops
+  end
+
+  def start(ary)
+    if ary.length != @ops.length
+      raise ArgumentError, "expect an array which lengths are #{@ops.length}"
+    end
+    @ops.map.with_index {|op, i|
+      op.start(ary[i])
+    }
+  end
+
+  def call(ary1, ary2)
+    if ary1.length != @ops.length || ary2.length != @ops.length
+      raise ArgumentError, "expect an array of arrays which lengths are #{@ops.length}"
+    end
+    @ops.zip(ary1, ary2).map {|op, v1, v2|
+      op.call(v1, v2)
+    }
+  end
+
+  def aggregate(ary)
+    if ary.length != @ops.length
+      raise ArgumentError, "expect an array which lengths are #{@ops.length}"
+    end
+    @ops.map.with_index {|op, i|
+      op.aggregate(ary[i])
+    }
+  end
+end

data/test/test_cmd_cat.rb

@@ -147,4 +147,44 @@ class TestTbCmdCat < Test::Unit::TestCase
     End
   end
 
+  def test_json_output
+    File.open(i1="i1.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
+      a,b,c
+      1,2,3
+      4,5,6
+    End
+    Tb::Cmd.main_cat(['-o', o="o.json", i1])
+    assert_equal(<<-"End".gsub(/\s+/, ''), File.read(o).gsub(/\s+/, ''))
+       [
+         {"a":"1", "b":"2", "c": "3"},
+         {"a":"4", "b":"5", "c": "6"}
+       ]
+    End
+  end
+
+  def test_json_output2
+    File.open(i1="i1.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
+      a,b,c
+      1,2,3
+      4,5,6
+    End
+    Tb::Cmd.main_cat(['-o', "json:" + (o="o.csv"), i1])
+    assert_equal(<<-"End".gsub(/\s+/, ''), File.read(o).gsub(/\s+/, ''))
+       [
+         {"a":"1", "b":"2", "c": "3"},
+         {"a":"4", "b":"5", "c": "6"}
+       ]
+    End
+  end
+
+  def test_invalid_output_format
+    File.open(i1="i1.csv", "w") {|f| f << <<-"End".gsub(/^[ \t]+/, '') }
+      a,b,c
+      1,2,3
+      4,5,6
+    End
+    exc = assert_raise(SystemExit) { Tb::Cmd.main_cat(['-o', "xson:o.csv", i1]) }
+    assert(!exc.success?)
+  end
+
 end

data/test/test_cmd_git_log.rb

@@ -15,6 +15,16 @@ class TestTbCmdGitLog < Test::Unit::TestCase
     FileUtils.rmtree @tmpdir
   end
 
+  def with_stderr(io)
+    save = $stderr
+    $stderr = io
+    begin
+      yield
+    ensure
+      $stderr = save
+    end
+  end
+
   def test_basic
     system("git init -q")
     File.open("foo", "w") {|f| f.puts "bar" }
@@ -43,4 +53,110 @@ class TestTbCmdGitLog < Test::Unit::TestCase
     assert_equal(filename, ftb.get_record(0)["filename"])
   end
 
+  def test_debug_git_log_output_input
+    system("git init -q")
+    File.open("foo", "w") {|f| f.puts "bar" }
+    system("git add foo")
+    system("git commit -q -m msg foo")
+    Tb::Cmd.main_git_log(['-o', o="o.csv", '--debug-git-log-output', g='gitlog'])
+    result = File.read(o)
+    tb = Tb.parse_csv(result)
+    assert_equal(1, tb.size)
+    assert_match(/,A,foo\n/, tb.get_record(0)["files"])
+    gresult = File.read(g)
+    assert(!gresult.empty?)
+    FileUtils.rmtree('.git')
+    Tb::Cmd.main_git_log(['-o', o="o.csv", '--debug-git-log-input', g])
+    result = File.read(o)
+    tb = Tb.parse_csv(result)
+    assert_equal(1, tb.size)
+    assert_match(/,A,foo\n/, tb.get_record(0)["files"])
+  end
+
+  def test_warn1
+    system("git init -q")
+    File.open("foo", "w") {|f| f.puts "bar" }
+    system("git add foo")
+    system("git commit -q -m msg foo")
+    Tb::Cmd.main_git_log(['-o', o="o.csv", '--debug-git-log-output', g='gitlog'])
+    result = File.read(o)
+    tb = Tb.parse_csv(result)
+    assert_equal(1, tb.size)
+    assert_match(/,A,foo\n/, tb.get_record(0)["files"])
+    gresult = File.binread(g)
+    FileUtils.rmtree('.git')
+    ###
+    gresult.sub!(/^:.*foo$/, ':hoge')
+    File.open(g, 'w') {|f| f.print gresult }
+    o2 = 'o2.csv'
+    File.open('log', 'w') {|log|
+      with_stderr(log) {
+        Tb::Cmd.main_git_log(['-o', o2, '--debug-git-log-input', g])
+      }
+    }
+    result = File.read(o2)
+    tb = Tb.parse_csv(result)
+    assert_equal(1, tb.size)
+    assert_not_match(/,A,foo\n/, tb.get_record(0)["files"])
+    log = File.read('log')
+    assert(!log.empty?)
+  end
+
+  def test_warn2
+    system("git init -q")
+    File.open("foo", "w") {|f| f.puts "bar" }
+    system("git add foo")
+    system("git commit -q -m msg foo")
+    Tb::Cmd.main_git_log(['-o', o="o.csv", '--debug-git-log-output', g='gitlog'])
+    result = File.read(o)
+    tb = Tb.parse_csv(result)
+    assert_equal(1, tb.size)
+    assert_match(/,A,foo\n/, tb.get_record(0)["files"])
+    gresult = File.binread(g)
+    FileUtils.rmtree('.git')
+    ###
+    gresult.sub!(/^author-name:/, 'author-name ')
+    File.open(g, 'w') {|f| f.print gresult }
+    o2 = 'o2.csv'
+    File.open('log', 'w') {|log|
+      with_stderr(log) {
+        Tb::Cmd.main_git_log(['-o', o2, '--debug-git-log-input', g])
+      }
+    }
+    result = File.read(o2)
+    tb = Tb.parse_csv(result)
+    assert_equal(1, tb.size)
+    assert_match(/,A,foo\n/, tb.get_record(0)["files"])
+    log = File.read('log')
+    assert(!log.empty?)
+  end
+
+  def test_warn3
+    system("git init -q")
+    File.open("foo", "w") {|f| f.puts "bar" }
+    system("git add foo")
+    system("git commit -q -m msg foo")
+    Tb::Cmd.main_git_log(['-o', o="o.csv", '--debug-git-log-output', g='gitlog'])
+    result = File.read(o)
+    tb = Tb.parse_csv(result)
+    assert_equal(1, tb.size)
+    assert_match(/,A,foo\n/, tb.get_record(0)["files"])
+    gresult = File.binread(g)
+    FileUtils.rmtree('.git')
+    ###
+    gresult.sub!(/end-commit/, 'endcommit')
+    File.open(g, 'w') {|f| f.print gresult }
+    o2 = 'o2.csv'
+    File.open('log', 'w') {|log|
+      with_stderr(log) {
+        Tb::Cmd.main_git_log(['-o', o2, '--debug-git-log-input', g])
+      }
+    }
+    result = File.read(o2)
+    tb = Tb.parse_csv(result)
+    assert_equal(0, tb.size)
+    log = File.read('log')
+    assert(!log.empty?)
+  end
+
 end