scout-gear 1.2.0 → 5.1.1

data/test/scout/indiferent_hash/test_options.rb

@@ -3,6 +3,16 @@ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1
 
 class TestOptions < Test::Unit::TestCase
 
+
+  def test_add_defaults
+    options = {:a => 1, "b" => 2}
+    assert_equal 2, IndiferentHash.add_defaults(options, :b => 3)["b"]
+    assert_equal 2, IndiferentHash.add_defaults(options, "b" => 3)["b"]
+    assert_equal 3, IndiferentHash.add_defaults(options, :c => 3)["c"]
+    assert_equal 3, IndiferentHash.add_defaults(options, "c" => 4)[:c]
+    assert_equal 3, IndiferentHash.add_defaults(options, "c" => 4)["c"]
+  end
+
   def test_positions2hash
     inputs = IndiferentHash.positional2hash([:one, :two, :three], 1, :two => 2, :four => 4)
     assert_equal 1, inputs[:one]
@@ -13,7 +23,7 @@ class TestOptions < Test::Unit::TestCase
 
   def test_process_to_hash
     list = [1,2,3,4]
-    assert_equal 4, IndiferentHash.process_to_hash(list){|l| l.collect{|e| e * 2}}[2]
+    assert_equal 4, IndiferentHash.process_to_hash(list){|l| l.collect{|e| e * 2 } }[2]
   end
 
   def test_hash2string

data/test/scout/misc/test_digest.rb

@@ -0,0 +1,30 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestMiscDigest < Test::Unit::TestCase
+  def test_digest_str
+    o = {:a => [1,2,3], :b => [1.1, 0.00001, 'hola']}
+    assert Misc.digest_str(o).include? "hola"
+  end
+
+  def test_digest_stream_located
+    TmpFile.with_file("TEST") do |filename|
+      Open.open(filename) do |f|
+        assert_equal 32, Misc.digest_str(f).length
+      end
+    end
+  end
+
+  def test_digest_stream_unlocated
+    TmpFile.with_file do |directory|
+      Path.setup(directory)
+      Open.write(directory.share.file, "TEST")
+      Misc.in_dir directory do
+        Open.open(Path.setup('share/file')) do |f|
+          assert_equal '\'share/file\'', Misc.digest_str(Path.setup('share/file'))
+        end
+      end
+    end
+  end
+end
+

data/test/scout/misc/test_filesystem.rb

@@ -0,0 +1,30 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestFilesystem < Test::Unit::TestCase
+  def test_in_dir
+    TmpFile.with_file do |tmpdir|
+      Misc.in_dir tmpdir do
+        assert_equal tmpdir, FileUtils.pwd
+      end
+    end
+  end
+
+  def test_relative
+    assert_equal 'bar', Misc.path_relative_to("/tmp/foo", "/tmp/foo/bar")
+    assert_equal 'bar/other', Misc.path_relative_to("/tmp/foo", "/tmp/foo/bar/other")
+
+    refute Misc.path_relative_to("/tmp/bar", "/tmp/foo/bar/other")
+    refute Misc.path_relative_to("/tmp/foo", "tmp/foo/bar/other")
+
+    TmpFile.with_file do |tmpdir|
+      Misc.in_dir tmpdir do
+        assert Misc.path_relative_to(tmpdir, "foo")
+        assert Misc.path_relative_to(tmpdir, File.join(tmpdir, "foo"))
+      end
+      assert Misc.path_relative_to(tmpdir, File.join(tmpdir, "foo"))
+      assert Misc.path_relative_to(File.dirname(tmpdir), File.join(tmpdir, "foo"))
+    end
+  end
+end
+

data/test/scout/misc/test_insist.rb

@@ -0,0 +1,13 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestMiscInsist < Test::Unit::TestCase
+  def test_insist
+    i = 0
+    Misc.insist do 
+      i += 1
+      raise "Not yet" if i < 3
+    end
+  end
+end
+

data/test/scout/open/test_lock.rb

@@ -0,0 +1,52 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tmpfile'
+
+class TestLock < Test::Unit::TestCase
+  def test_locks
+    num = 10
+    TmpFile.with_file do |lockfile|
+      TmpFile.with_file do |output|
+        f = File.open(output, 'w') 
+        thrs = num.times.collect do 
+          Thread.new do
+            Open.lock lockfile, true, min_sleep: 0.01, max_sleep: 0.05, sleep_inc: 0.001 do
+              f.write "["
+              sleep 0.01
+              f.write "]"
+            end
+          end
+        end
+        thrs.each{|t| t.join }
+        f.close
+        assert_equal "[]" * num, File.open(output).read 
+      end
+    end
+  end
+
+  def test_keep_locked
+    num = 10
+    TmpFile.with_file do |lockfile|
+      TmpFile.with_file do |output|
+        f = File.open(output, 'w') 
+        thrs = num.times.collect do 
+          Thread.new do
+            lock = Lockfile.new(lockfile, min_sleep: 0.01, max_sleep: 0.05, sleep_inc: 0.001)
+            res = Open.lock lock do
+              f.write "["
+              raise KeepLocked, "1"
+            end
+            f.write res
+            f.write "]"
+            lock.unlock
+          end
+        end
+        thrs.each{|t| t.join }
+        f.close
+        assert_equal "[1]" * num, File.open(output).read 
+      end
+    end
+  end
+end
+

data/test/scout/open/test_remote.rb

@@ -0,0 +1,25 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/open'
+
+class TestOpenRemote < Test::Unit::TestCase
+  def test_wget
+    teardown
+    5.times do
+      assert(Misc.fixutf8(Open.wget('http://google.com', :quiet => true, :nocache => true).read) =~ /html/)
+      assert(Misc.fixutf8(Open.wget('http://google.com', :quiet => true).read) =~ /html/)
+    end
+  end
+
+  def test_ssh
+    TmpFile.with_file("TEST") do |f|
+      begin
+        assert_equal "TEST", Open.ssh("ssh://localhost:#{f}").read
+      rescue ConcurrentStreamProcessFailed
+        raise $! unless $!.message.include? "Connection refused"
+      end
+    end if false
+  end
+end
+

data/test/scout/open/test_stream.rb

@@ -0,0 +1,515 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/path'
+require 'scout/open'
+
+class TestOpenStream < Test::Unit::TestCase
+  def test_consume_stream
+    content =<<-EOF
+1
+2
+3
+4
+    EOF
+    TmpFile.with_file(content) do |file|
+      TmpFile.with_file do |target|
+        f = File.open(file)
+        Open.consume_stream(f, false, target)
+        assert_equal content, File.open(target).read
+      end
+    end
+  end
+
+  def test_sensible_write
+    content =<<-EOF
+1
+2
+3
+4
+    EOF
+    TmpFile.with_file(content) do |file|
+      TmpFile.with_file do |target|
+        f = File.open(file)
+        Open.sensible_write(target, f)
+        assert_equal content, File.open(target).read
+      end
+    end
+  end
+
+  def test_open_pipe
+    sout = Open.open_pipe do |sin|
+      10.times do |i|
+        sin.puts "line #{i}"
+      end
+    end
+
+    TmpFile.with_file do |tmp|
+      Open.consume_stream(sout, false, tmp)
+      assert_equal 10, Open.read(tmp).split("\n").length
+    end
+  end
+
+  def test_open_pipe_and_close
+
+    10000.times do
+      sout = Open.open_pipe do |sin|
+        sin.puts "hola"
+      end
+      sout.read
+      sout.close
+    end
+  end
+
+
+  def test_open_pipe_error
+    sout = Open.open_pipe do |sin|
+      10.times do |i|
+        sin.puts "line #{i}"
+      end
+      raise ScoutException
+    end
+
+    TmpFile.with_file do |tmp|
+      assert_raise ScoutException do
+        Open.consume_stream(sout, false, tmp)
+      end
+    end
+  end
+
+  def test_tee_stream
+    num = 2000
+    sout = Open.open_pipe do |sin|
+      num.times do |i|
+        sin.puts "line #{i} - #{rand(100000).to_s * 100}"
+      end
+    end
+
+    TmpFile.with_file do |tmp|
+      Path.setup tmp
+
+      s1, s2 = Open.tee_stream_thread sout
+
+      t1 = Open.consume_stream(s1, true, tmp.file1)
+      t2 = Open.consume_stream(s2, true, tmp.file2)
+      t1.join
+      t2.join
+      assert_equal num, Open.read(tmp.file1).split("\n").length
+      assert_equal num, Open.read(tmp.file2).split("\n").length
+    end
+  end
+
+  def test_tee_stream_source_error
+    5.times do |i|
+      num = 2000
+      sout = Open.open_pipe do |sin|
+        num.times do |i|
+          sin.puts "line #{i} - #{rand(100000).to_s * 100}"
+        end
+        raise ScoutException
+      end
+
+      TmpFile.with_file do |tmp|
+        Path.setup tmp
+
+        s1, s2 = Open.tee_stream_thread sout
+
+        t1 = Open.consume_stream(s1, true, tmp.file1)
+        t2 = Open.consume_stream(s2, true, tmp.file2)
+        assert_raise ScoutException do
+          begin
+            t1.join
+            t2.join
+          rescue 
+            raise
+          ensure
+            [t1, t2].each do |t|
+              begin
+                t.join if t.alive?
+              rescue ScoutException
+              end
+            end
+          end
+        end
+        refute Open.exist?(tmp.file1)
+        refute Open.exist?(tmp.file2)
+      end
+    end
+  end
+
+  def test_tee_stream_save_error
+    Log.with_severity 6 do
+      5.times do |i|
+        TmpFile.with_file do |tmp|
+          Path.setup tmp
+          assert_raise ScoutException do
+            num = 2000
+            begin
+              sout = Open.open_pipe do |sin|
+                num.times do |i|
+                  sin.puts "line #{i} - #{rand(100000).to_s * 100}"
+                end
+              end
+
+              begin
+                s1, s2 = Open.tee_stream_thread sout
+
+                s2.abort ScoutException.new
+                t = Open.consume_stream(s1, true, tmp.file)
+
+                t.join
+              ensure
+                s1.close if s1.respond_to?(:close) && ! s1.closed?
+                s1.join if s1.respond_to?(:join) && ! s1.joined?
+                s2.close if s2.respond_to?(:close) && ! s2.closed?
+                s2.join if s2.respond_to?(:join) && ! s2.joined?
+              end
+            ensure
+              sout.close if sout.respond_to?(:close) && ! sout.closed?
+              sout.join if sout.respond_to?(:join) && ! sout.joined?
+            end
+          end
+          refute Open.exist?(tmp.file)
+        end
+      end
+    end
+  end
+
+  #  def test_collapse_stream
+  #    text=<<-EOF
+  #row1 A B C
+  #row1 a b c
+  #row2 AA BB CC
+  #row2 aa bb cc
+  #    EOF
+  #
+  #    s = StringIO.new text
+  #    tsv = TSV.open Misc.collapse_stream(s,nil, " "), :sep => " " 
+  #    assert_equal ["A", "a"], tsv["row1"][0]
+  #    assert_equal ["BB", "bb"], tsv["row2"][1]
+  #  end
+  #
+  #  def test_collapse_sum
+  #    text=<<-EOF
+  #row1 12
+  #row1 4
+  #row2 10
+  #row2 6
+  #    EOF
+  #
+  #    s = StringIO.new text
+  #    stream = Misc.collapse_stream(s,nil, " ") do |parts|
+  #      next nil if parts.empty?
+  #      parts.first.split("|").collect{|p| p.to_f}.inject(0){|acc,e| acc += e}.to_s
+  #    end
+  #    tsv = TSV.open  stream, :sep => " " 
+  #    ppp tsv.to_s
+  #  end
+  #
+  #  def test_collapse_stream_gap
+  #    text=<<-EOF
+  #row2 AA BB 
+  #row2 aa bb cc
+  #    EOF
+  #
+  #    s = StringIO.new text
+  #    assert Misc.collapse_stream(s, nil, " ").read =~  /\|cc$/
+  #    
+  #    text=<<-EOF
+  #row2 aa bb cc
+  #row2 AA BB 
+  #    EOF
+  #
+  #    s = StringIO.new text
+  #    assert Misc.collapse_stream(s, nil, " ").read =~  /cc\|$/
+  #
+  #    text=<<-EOF
+  #row2 AA BB
+  #row2 aa bb cc
+  #    EOF
+  #
+  #    s = StringIO.new text
+  #    assert Misc.collapse_stream(s, nil, " ").read =~  /\|cc$/
+  #
+  #  end
+  #
+  #
+  #  def test_paste_stream
+  #    text1=<<-EOF
+  #row1 A B C
+  #row2 AA BB CC
+  #row3 AAA BBB CCC
+  #    EOF
+  #
+  #    text2=<<-EOF
+  #row1 a b
+  #row2 aa bb
+  #    EOF
+  #
+  #    text3=<<-EOF
+  #row1 c
+  #row2 cc
+  #row3 ccc
+  #    EOF
+  #
+  #    s1 = StringIO.new text1
+  #    s2 = StringIO.new text2
+  #    s3 = StringIO.new text3
+  #    tsv = TSV.open Misc.paste_streams([s1,s2,s3], nil, " "), :sep => " ", :type => :list
+  #    assert_equal ["A", "B", "C", "a", "b", "c"], tsv["row1"]
+  #    assert_equal ["AA", "BB", "CC", "aa", "bb", "cc"], tsv["row2"]
+  #    assert_equal ["AAA", "BBB", "CCC", "", "", "ccc"], tsv["row3"]
+  #  end
+  #
+  #  def test_sort_stream
+  #    text =<<-EOF
+  ###
+  ###
+  ###
+  ##Row LabelA LabelB LabelC
+  #row2 AA BB CC
+  #row3 AAA BBB CCC
+  #row1 A B C
+  #    EOF
+  #    s = StringIO.new text
+  #    sorted = Misc.sort_stream(s)
+  #    assert_equal %w(## ## ## #Row row2 row3 row1), text.split("\n").collect{|l| l.split(" ").first}
+#    assert_equal %w(## ## ## #Row row1 row2 row3), sorted.read.split("\n").collect{|l| l.split(" ").first}
+#  end
+#
+#  def test_sort_long_stream
+#    text =<<-EOF
+###
+###
+###
+##Row LabelA LabelB LabelC
+#row2 AA BB CC
+#row3 AAA BBB CCC
+#row1 A B C
+#    EOF
+#
+#    s = StringIO.new text + (text.split("\n")[-3..-1] * "\n" + "\n") * 10000
+#    sorted = Misc.sort_stream(s)
+#    assert_equal %w(## ## ## #Row row2 row3 row1), text.split("\n").collect{|l| l.split(" ").first}
+#    assert_equal %w(## ## ## #Row row1 row2 row3), sorted.read.split("\n").collect{|l| l.split(" ").first}
+#  end
+#
+#  def test_sort_stream2
+#    text =<<-EOF
+###
+###
+###
+##Row LabelA LabelB LabelC
+#row2 AA BB CC
+#row3 AAA BBB CCC
+#row1 A B C
+#    EOF
+#    s = StringIO.new text
+#    sorted = Misc.sort_stream(Misc.sort_stream(s))
+#    assert_equal %w(## ## ## #Row row2 row3 row1), text.split("\n").collect{|l| l.split(" ").first}
+#    assert_equal %w(## ## ## #Row row1 row2 row3), sorted.read.split("\n").collect{|l| l.split(" ").first}
+#  end
+#
+#  def test_dup_stream
+#    text =<<-EOF
+##: :sep=" "
+##Row LabelA LabelB LabelC
+#row2 AA BB CC
+#row3 AAA BBB CCC
+#row1 A B C
+#    EOF
+#
+#    text = text * 10000
+#    TmpFile.with_file(text) do |tmp|
+#      io = Open.open(tmp)
+#      dup = Misc.dup_stream(io)
+#      new_text = StringIO.new ""
+#      Misc.consume_stream dup, true, new_text
+#      Misc.consume_stream io, false
+#      new_text.rewind
+#      assert_equal text, new_text.read
+#    end
+#
+#
+#    TmpFile.with_file(text) do |tmp|
+#      io = Open.open(tmp)
+#      dup = Misc.dup_stream(io)
+#      Misc.consume_stream dup, true
+#      assert_equal text, io.read
+#    end
+#  end
+#
+#  def test_dup_stream_multiple
+#    text =<<-EOF
+#row2 AA BB CC
+#row3 AAA BBB CCC
+#row1 A B C
+#    EOF
+#
+#    text = text * 10000
+#    num = 5
+#    str_io = StringIO.new
+#    strs = []
+#    num.times{ strs << StringIO.new }
+#    TmpFile.with_file(text) do |tmp|
+#      io = Open.open(tmp)
+#      copies = Misc.dup_stream_multiple(io, num)
+#
+#
+#      copies.each_with_index do |cio,i|
+#        cio.add_callback do 
+#          str = strs[i] 
+#          str.rewind
+#          assert_equal text, str.read
+#        end
+#        Misc.consume_stream cio, true, strs[i], false
+#      end
+#
+#
+#      Misc.consume_stream io, false, str_io, false
+#      str_io.rewind
+#      assert_equal text, str_io.read
+#
+#    end
+#  end
+#
+#  def test_remove_lines
+#    text1 =<<-EOF
+#line1
+#line2
+#line3
+#line4
+#    EOF
+#    text2 =<<-EOF
+#line3
+#line1
+#    EOF
+#
+#    TmpFile.with_file(text1) do |file1|
+#      TmpFile.with_file(text2) do |file2|
+#        assert ! Misc.remove_lines(file1, file2, true).read.split("\n").include?("line1")
+#        assert Misc.remove_lines(file1, file2, true).read.split("\n").include?("line2")
+#      end
+#    end
+#  end
+#
+#
+#
+#  def test_select_lines
+#    text1 =<<-EOF
+#line1
+#line2
+#line3
+#line4
+#    EOF
+#    text2 =<<-EOF
+#line3
+#line1
+#line5
+#    EOF
+#
+#    TmpFile.with_file(text1) do |file1|
+#      TmpFile.with_file(text2) do |file2|
+#        assert Misc.select_lines(file1, file2, true).read.split("\n").include?("line1")
+#        assert ! Misc.select_lines(file1, file2, true).read.split("\n").include?("line2")
+#      end
+#    end
+#  end
+#
+#  def test_consume_into_string_io
+#    text =<<-EOF
+#line1
+#line2
+#line3
+#line4
+#    EOF
+#
+#    TmpFile.with_file(text) do |file|
+#      out = StringIO.new
+#      io = Open.open(file)
+#      Misc.consume_stream(io, false, out, false)
+#      out.rewind
+#      assert_equal text, out.read
+#    end
+#  end
+#
+#  def test_monitor
+#    text =<<-EOF
+#line1
+#line2
+#line3
+#line4
+#    EOF
+#
+#    TmpFile.with_file(text) do |file|
+#      io = Open.open(file)
+#      lines = Set.new
+#      io2 = Misc.line_monitor_stream io do |line|
+#        lines << line
+#      end
+#      Misc.consume_stream(io2, false)
+#      assert_equal text, lines.to_a * ""
+#    end
+#  end
+#
+#  def test_sort
+#    assert_raise RbbtException do
+#      io = Misc.open_pipe do |sin|
+#        sin.puts "#START"
+#        20.times do
+#          sin.puts rand(1000).to_s
+#          sleep 0.1
+#        end
+#        raise RbbtException
+#      end
+#
+#      sio = Misc.sort_stream(io)
+#      begin
+#        Misc.consume_stream(sio, false, STDOUT)
+#      rescue
+#        raise $!
+#      end
+#    end
+#  end
+#
+#  def test_gz_pipe
+#    text =<<-EOF
+#line1
+#line2
+#line3
+#line4
+#    EOF
+#
+#    TmpFile.with_file nil, :extension => 'txt.gz' do |file|
+#      sout = Misc.open_gz_pipe do |sin|
+#        text.split("\n").each do |line|
+#          sin.puts line
+#        end
+#      end
+#
+#      Open.mkdir File.basename(file)
+#      thr1 = Misc.consume_stream(sout, true, file)
+#      thr1.join
+#      assert Open.gzip?(file)
+#      assert_equal text, Open.read(file)
+#    end
+#  end
+#
+#  def test_open_pipe_error
+#    sout = Misc.open_pipe do |sin|
+#      10.times do |i|
+#        sin.puts "line #{i}"
+#      end
+#      raise
+#    end
+#
+#    TmpFile.with_file do |tmp|
+#      #Misc.consume_stream(sout, false, tmp)
+#      assert_raise do
+#        Open.write(tmp, sout)
+#      end
+#    end
+#  end
+#end
+end
+