scout-gear 7.1.0 → 7.3.0

data/test/scout/open/test_stream.rb

@@ -5,7 +5,7 @@ require 'scout/path'
 require 'scout/open'
 
 class TestOpenStream < Test::Unit::TestCase
-  def test_consume_stream
+  def test_stream
     content =<<-EOF
 1
 2
@@ -99,6 +99,32 @@ class TestOpenStream < Test::Unit::TestCase
     end
   end
 
+  def test_tee_stream_multiple
+    num = 2000
+    sout = Open.open_pipe do |sin|
+      num.times do |i|
+        sin.puts "line #{i} - #{rand(100000).to_s * 100}"
+      end
+    end
+
+    TmpFile.with_file do |tmp|
+      Path.setup tmp
+
+      s1, s2, s3 = Open.tee_stream_thread_multiple sout, 3
+
+      t1 = Open.consume_stream(s1, true, tmp.file1)
+      t2 = Open.consume_stream(s2, true, tmp.file2)
+      t3 = Open.consume_stream(s3, true, tmp.file3)
+      t1.join
+      t2.join
+      t3.join
+      assert_equal num, Open.read(tmp.file1).split("\n").length
+      assert_equal num, Open.read(tmp.file2).split("\n").length
+      assert_equal num, Open.read(tmp.file3).split("\n").length
+    end
+  end
+
+
   def test_tee_stream_source_error
     5.times do |i|
       num = 2000
@@ -137,6 +163,47 @@ class TestOpenStream < Test::Unit::TestCase
     end
   end
 
+  def test_tee_stream_source_error_multiple
+    5.times do |i|
+      num = 2000
+      sout = Open.open_pipe do |sin|
+        num.times do |i|
+          sin.puts "line #{i} - #{rand(100000).to_s * 100}"
+        end
+        raise ScoutException
+      end
+
+      TmpFile.with_file do |tmp|
+        Path.setup tmp
+
+        s1, s2, s3 = Open.tee_stream_thread_multiple sout, 3
+
+        t1 = Open.consume_stream(s1, true, tmp.file1)
+        t2 = Open.consume_stream(s2, true, tmp.file2)
+        t3 = Open.consume_stream(s3, true, tmp.file3)
+        assert_raise ScoutException do
+          begin
+            t1.join
+            t2.join
+            t3.join
+          rescue 
+            raise
+          ensure
+            [t1, t2, t2].each do |t|
+              begin
+                t.join if t.alive?
+              rescue ScoutException
+              end
+            end
+          end
+        end
+        refute Open.exist?(tmp.file1)
+        refute Open.exist?(tmp.file2)
+        refute Open.exist?(tmp.file3)
+      end
+    end
+  end
+
   def test_tee_stream_save_error
     Log.with_severity 6 do
       50.times do |i|
@@ -175,6 +242,98 @@ class TestOpenStream < Test::Unit::TestCase
     end
   end
 
+  def test_tee_stream_save_error_multiple
+    Log.with_severity 6 do
+      50.times do |i|
+        TmpFile.with_file do |tmp|
+          Path.setup tmp
+          assert_raise ScoutException do
+            num = 2000
+            begin
+              sout = Open.open_pipe do |sin|
+                num.times do |i|
+                  sin.puts "line #{i} - #{rand(100000).to_s * 100}"
+                end
+              end
+
+              begin
+                s1, s2, s3 = Open.tee_stream_thread_multiple sout, 3
+
+                s2.abort ScoutException.new
+                t1 = Open.consume_stream(s1, true, tmp.file1)
+                t2 = Open.consume_stream(s2, true, tmp.file2)
+
+                t1.join
+                t2.join
+              ensure
+                s1.close if s1.respond_to?(:close) && ! s1.closed?
+                s1.join if s1.respond_to?(:join) && ! s1.joined?
+                s2.close if s2.respond_to?(:close) && ! s2.closed?
+                s2.join if s2.respond_to?(:join) && ! s2.joined?
+                s3.close if s3.respond_to?(:close) && ! s3.closed?
+                s3.join if s3.respond_to?(:join) && ! s3.joined?
+              end
+            ensure
+              sout.close if sout.respond_to?(:close) && ! sout.closed?
+              sout.join if sout.respond_to?(:join) && ! sout.joined?
+            end
+          end
+          refute Open.exist?(tmp.file1)
+          refute Open.exist?(tmp.file2)
+        end
+      end
+    end
+  end
+
+  def test_sort_stream
+    text =<<-EOF
+##
+##
+##
+#Row LabelA LabelB LabelC
+row2 AA BB CC
+row3 AAA BBB CCC
+row1 A B C
+    EOF
+    s = StringIO.new text
+    sorted = Open.sort_stream(s)
+    assert_equal %w(## ## ## #Row row2 row3 row1), text.split("\n").collect{|l| l.split(" ").first}
+       assert_equal %w(## ## ## #Row row1 row2 row3), sorted.read.split("\n").collect{|l| l.split(" ").first}
+  end
+
+  def test_sort_long_stream
+    text =<<-EOF
+##
+##
+##
+#Row LabelA LabelB LabelC
+row2 AA BB CC
+row3 AAA BBB CCC
+row1 A B C
+    EOF
+
+    s = StringIO.new text + (text.split("\n")[-3..-1] * "\n" + "\n") * 10000
+    sorted = Open.sort_stream(s)
+    assert_equal %w(## ## ## #Row row2 row3 row1), text.split("\n").collect{|l| l.split(" ").first}
+    assert_equal %w(## ## ## #Row row1 row2 row3), sorted.read.split("\n").collect{|l| l.split(" ").first}
+  end
+
+  def test_sort_stream2
+    text =<<-EOF
+##
+##
+##
+#Row LabelA LabelB LabelC
+row2 AA BB CC
+row3 AAA BBB CCC
+row1 A B C
+    EOF
+    s = StringIO.new text
+    sorted = Open.sort_stream(Open.sort_stream(s))
+    assert_equal %w(## ## ## #Row row2 row3 row1), text.split("\n").collect{|l| l.split(" ").first}
+    assert_equal %w(## ## ## #Row row1 row2 row3), sorted.read.split("\n").collect{|l| l.split(" ").first}
+  end
+
   #  def test_collapse_stream
   #    text=<<-EOF
   #row1 A B C

data/test/scout/path/test_find.rb

@@ -29,7 +29,7 @@ class TestPathFind < Test::Unit::TestCase
   def test_current
     path = Path.setup("share/data/some_file", 'scout')
     TmpFile.in_dir do |tmpdir|
-      assert_equal File.join(tmpdir,"share/data/some_file"),  path.find(:current)
+      assert_equal_path File.join(tmpdir,"share/data/some_file"),  path.find(:current)
     end
   end
 
@@ -38,7 +38,7 @@ class TestPathFind < Test::Unit::TestCase
     TmpFile.in_dir do |tmpdir|
       FileUtils.mkdir_p(File.dirname(File.join(tmpdir, path)))
       File.write(File.join(tmpdir, path), 'string')
-      assert_equal File.join(tmpdir,"share/data/some_file"),  path.find
+      assert_equal_path File.join(tmpdir,"share/data/some_file"),  path.find
       assert_equal :current,  path.find.where
       assert_equal "share/data/some_file",  path.find.original
     end
@@ -67,7 +67,7 @@ class TestPathFind < Test::Unit::TestCase
 
     p = Path.setup("/tmp/foo/bar")
     assert p.located?
-    assert_equal p, p.find
+    assert_equal_path p, p.find
     
   end
 
@@ -75,10 +75,10 @@ class TestPathFind < Test::Unit::TestCase
     path = Path.setup("share/data/some_file", 'scout')
     TmpFile.with_file do |tmpdir|
       path.path_maps[:custom] = [tmpdir, '{PATH}'] * "/"
-      assert_equal File.join(tmpdir,"share/data/some_file"),  path.find(:custom)
+      assert_equal_path File.join(tmpdir,"share/data/some_file"),  path.find(:custom)
 
       path.path_maps[:custom] = [tmpdir, '{TOPLEVEL}/{PKGDIR}/{SUBPATH}'] * "/"
-      assert_equal File.join(tmpdir,"share/scout/data/some_file"),  path.find(:custom)
+      assert_equal_path File.join(tmpdir,"share/scout/data/some_file"),  path.find(:custom)
     end
   end
 
@@ -87,7 +87,7 @@ class TestPathFind < Test::Unit::TestCase
     TmpFile.with_file do |tmpdir|
       path.pkgdir = 'scout_alt'
       path.path_maps[:custom] = [tmpdir, '{TOPLEVEL}/{PKGDIR}/{SUBPATH}'] * "/"
-      assert_equal File.join(tmpdir,"share/scout_alt/data/some_file"),  path.find(:custom)
+      assert_equal_path File.join(tmpdir,"share/scout_alt/data/some_file"),  path.find(:custom)
     end
   end
 
@@ -98,6 +98,13 @@ class TestPathFind < Test::Unit::TestCase
     assert_equal "/some_dir/scout_commands/find",  Path.follow(path, '/some_dir/{PATH/bin\/scout/scout_commands}')
   end
 
-
+  def test_gz
+    TmpFile.with_file do |tmpdir|
+      Path.setup(tmpdir)
+      Open.write(tmpdir.somefile + '.gz', "FOO")
+      assert_equal_path tmpdir.somefile + '.gz', tmpdir.somefile.find
+      assert Open.exist?(tmpdir.somefile)
+    end
+  end
 end
 

data/test/scout/resource/test_software.rb

@@ -0,0 +1,24 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestResourceSoftware < Test::Unit::TestCase
+  module TestResource
+    extend Resource
+
+    self.subdir = Path.setup('tmp/test-resource')
+  end
+
+  def test_install
+    Resource.install nil, "scout_install_example", tmpdir.software do
+      <<-EOF
+echo "#!/bin/bash\necho WORKING" > $OPT_BIN_DIR/scout_install_example
+chmod +x $OPT_BIN_DIR/scout_install_example
+      EOF
+    end
+    assert_nothing_raised do
+      CMD.cmd(tmpdir.software.opt.bin.scout_install_example).read
+    end
+    assert_equal "WORKING", CMD.cmd('scout_install_example').read.strip
+  end
+end
+

data/test/scout/test_config.rb

@@ -0,0 +1,66 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestConfig < Test::Unit::TestCase
+  setup do
+    Scout::Config.set({:cpus => 30}, :test_config, :test)
+    Scout::Config.set(:cpus, 5, "slow::2", :test)
+    Scout::Config.set({:token => "token"}, "token")
+    Scout::Config.set(:notoken, "no_token")
+    Scout::Config.set({:emptytoken => "empty"})
+  end
+
+  def test_simple
+    assert_equal 30, Scout::Config.get(:cpus, :test_config)
+  end
+
+  def test_match
+    assert_equal({20 => ["token"]}, Scout::Config.match({["key:token"] => "token"}, "key:token"))
+  end
+
+  def test_simple_no_token
+    assert_equal "token", Scout::Config.get("token", "token")
+    assert_equal "no_token", Scout::Config.get("notoken", "key:notoken")
+    assert_equal 'token', Scout::Config.get("token", "key:token")
+    assert_equal 'token', Scout::Config.get("token")
+    assert_equal nil, Scout::Config.get("token", "someotherthing")
+    assert_equal "default_token", Scout::Config.get("token", 'unknown', :default => 'default_token')
+    assert_equal 'empty', Scout::Config.get("emptytoken", 'key:emptytoken')
+  end
+
+  def test_prio
+    assert_equal 5, Scout::Config.get(:cpus, :slow, :test)
+  end
+
+  def test_with_config
+    Scout::Config.add_entry 'key', 'valueA', 'token'
+    assert_equal "valueA", Scout::Config.get('key', 'token')
+    assert_equal "default", Scout::Config.get('key2', 'token', :default => 'default')
+
+    Scout::Config.with_config do 
+      Scout::Config.add_entry 'key', 'valueB', 'token'
+      Scout::Config.add_entry 'key2', 'valueB2', 'token'
+      assert_equal "valueB", Scout::Config.get('key', 'token')
+      assert_equal "valueB2", Scout::Config.get('key2', 'token', :default => 'default')
+    end
+
+    assert_equal "valueA", Scout::Config.get('key', 'token')
+    assert_equal "default", Scout::Config.get('key2', 'token', :default => 'default')
+  end
+
+  def test_order
+    Scout::Config.add_entry 'key', 'V1', 'token1'
+    Scout::Config.add_entry 'key', 'V2', 'token2'
+    Scout::Config.add_entry 'key', 'V3', 'token2'
+
+    assert_equal "V3", Scout::Config.get('key', 'token2')
+    assert_equal "V1", Scout::Config.get('key', 'token1')
+    assert_equal "V3", Scout::Config.get('key', 'token2', 'token1')
+    assert_equal "V1", Scout::Config.get('key', 'token1', 'token2')
+  end
+
+  def test_default
+    Scout::Config.add_entry 'key', 'V1', 'token1'
+    assert_equal "V3", Scout::Config.get('key', 'token2', :default => 'V3')
+  end
+end

data/test/scout/test_meta_extension.rb

@@ -19,6 +19,16 @@ class TestMetaExtension < Test::Unit::TestCase
     assert_equal :code, str2.code
   end
 
+  def test_marshal
+    str = "String"
+    ExtensionClass.setup(str, :code)
+    assert ExtensionClass === str
+    assert_equal :code, str.code
+
+    str2 = Marshal.load(Marshal.dump(str))
+    assert_equal :code, str2.code
+  end
+
   def test_setup_alternatives
     str = "String"
 

data/test/scout/test_named_array.rb

@@ -0,0 +1,19 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestNamedArray < Test::Unit::TestCase
+  def test_identify_names
+    names =<<-EOF.split("\n")
+ValueA
+ValueB (Entity type)
+15
+    EOF
+    assert_equal 0, NamedArray.identify_name(names, "ValueA")
+    assert_equal :key, NamedArray.identify_name(names, :key)
+    assert_equal 0, NamedArray.identify_name(names, nil)
+    assert_equal 1, NamedArray.identify_name(names, "ValueB (Entity type)")
+    assert_equal 1, NamedArray.identify_name(names, "ValueB")
+    assert_equal 1, NamedArray.identify_name(names, 1)
+  end
+end
+

data/test/scout/test_persist.rb

@@ -56,6 +56,21 @@ class TestPersist < Test::Unit::TestCase
     end
   end
 
+  def test_stream_multiple
+    TmpFile.with_file do |tmpfile|
+      Path.setup(tmpfile)
+      obj = "TEST\nTEST"
+      stream = StringIO.new obj
+      stream.rewind
+      res1 = Persist.persist(tmpfile, :string, :dir => tmpdir.persist, :tee_copies => 2){ stream }
+      res2 = res1.next
+      assert IO === res1
+      assert_equal obj, res1.read
+      assert_equal obj, res2.read
+      assert_equal obj, Persist.persist(tmpfile, :string, :dir => tmpdir.persist){ raise ScoutException }
+    end
+  end
+
   def test_update_time
     TmpFile.with_file do |dir|
       Path.setup(dir)
@@ -79,5 +94,86 @@ class TestPersist < Test::Unit::TestCase
       assert_equal "TEST2", Persist.persist(dir.cache, type, :dir => tmpdir.persist, :update => dir.file3){ Open.read(dir.file3) }
     end
   end
+
+  def test_concurrent
+    num = 10
+
+    s = 0.01
+    10.times do
+      TmpFile.with_file do |file|
+        output1 = file + '.output1'
+        output2 = file + '.output2'
+        pid1 = Process.fork do
+          Open.purge_pipes
+          sleep rand/10.0
+          io = Persist.persist("test", :string, :path => file) do
+            Open.open_pipe do |sin|
+              num.times do |i|
+                sin.puts "line-#{i}-#{Process.pid}"
+                sleep s
+              end
+            end
+          end
+          if IO === io
+            Open.consume_stream(io, false)
+          else
+            Open.write(output1, io)
+          end
+        end
+        pid2 = Process.fork do
+          Open.purge_pipes
+          sleep rand/10.0
+          io = Persist.persist("test", :string, :path => file) do
+            Open.open_pipe do |sin|
+              num.times do |i|
+                sin.puts "line-#{i}-#{Process.pid}"
+                sleep s
+              end
+            end
+          end
+          if IO === io
+            Open.consume_stream(io, false)
+          else
+            Open.write(output2, io)
+          end
+        end
+        Process.wait
+        Process.wait
+
+        assert File.exist?(output1) || File.exist?(output2)
+        [pid1, pid2].zip([output2, output1]).each do |pid, found|
+          next unless File.exist?(found)
+          assert Open.read(found).include? "-#{pid}\n"
+        end
+        [pid1, pid2].zip([output1, output2]).each do |pid, found|
+          next unless File.exist?(found)
+          refute Open.read(found).include? "-#{pid}\n"
+        end
+        Open.rm file
+        Open.rm output1
+        Open.rm output2
+      end
+    end
+  end
+
+  def __test_speed
+    times = 100_000
+    TmpFile.with_file do |tmpfile|
+      sout = Persist.persist(tmpfile, :string, :path => tmpfile) do
+        Open.open_pipe do |sin|
+          times.times do |i|
+            sin.puts "line-#{i}"
+          end
+        end
+      end
+
+      Log::ProgressBar.with_bar do |bar|
+        while l = sout.gets
+          bar.tick
+        end
+      end
+    end
+  end
+
 end
 

data/test/scout/test_tmpfile.rb

@@ -45,7 +45,7 @@ class TestTmpFile < Test::Unit::TestCase
 
   def test_in_dir
     TmpFile.in_dir do |dir|
-      assert_equal dir, FileUtils.pwd
+      assert_equal_path dir, FileUtils.pwd
     end
   end
 

data/test/scout/test_tsv.rb

@@ -20,7 +20,7 @@ row4    a    a    id3
     EOF
 
     tsv = TmpFile.with_file(content) do |filename|
-      TSV.open(filename)
+      TSV.open(filename, :persist => false)
     end
 
     TmpFile.with_file(content2) do |filename|
@@ -30,5 +30,54 @@ row4    a    a    id3
     assert_include tsv.keys, 'row4'
     assert_include tsv.keys, 'row1'
   end
+
+  def test_open_persist
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:double#:merge=:concat
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+row2    a    a    id3
+    EOF
+
+    tsv = TmpFile.with_file(content) do |filename|
+      TSV.open(filename, :persist => true)
+    end
+
+    assert tsv.respond_to?(:persistence_class)
+    assert_equal TokyoCabinet::HDB, tsv.persistence_class
+
+    assert_include tsv.keys, 'row1'
+    assert_include tsv.keys, 'row2'
+  end
+
+  def test_open_persist_in_situ
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:double#:merge=:concat
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+row2    a    a    id3
+    EOF
+
+    tsv = TmpFile.with_file(content) do |filename|
+      TSV.open(filename, :persist => false)
+    end
+
+    assert_include tsv.keys, 'row1'
+    assert_include tsv.keys, 'row2'
+    assert_equal %w(A a), tsv["row2"][0]
+
+    tsv = TmpFile.with_file(content) do |filename|
+      TSV.open(filename, :persist => true, :merge => true)
+    end
+
+    assert tsv.respond_to?(:persistence_class)
+    assert_equal TokyoCabinet::HDB, tsv.persistence_class
+
+    assert_include tsv.keys, 'row1'
+    assert_include tsv.keys, 'row2'
+    assert_equal %w(A a), tsv["row2"][0]
+  end
 end
 

data/test/scout/test_work_queue.rb

@@ -3,7 +3,7 @@ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1
 
 require 'scout/log'
 class TestWorkQueue < Test::Unit::TestCase
-  def test_queue_remove_workers
+  def test_a_queue_remove_workers
     num = 10
     reps = 10_000
     q = WorkQueue.new num do |obj|
@@ -19,7 +19,9 @@ class TestWorkQueue < Test::Unit::TestCase
       q.write i
     end
 
-    num.times do q.remove_one_worker end
+    (num - 1).times do q.remove_one_worker end
+
+    Thread.pass until q.workers.length == 1
 
     w = q.add_worker do |obj|
       "HEY"
@@ -38,7 +40,7 @@ class TestWorkQueue < Test::Unit::TestCase
 
   def test_queue
     num = 10
-    reps = 10_000
+    reps = 1_000
     q = WorkQueue.new num do |obj|
       [Process.pid.to_s, obj.to_s] * " "
     end
@@ -55,8 +57,8 @@ class TestWorkQueue < Test::Unit::TestCase
     end
 
     Process.wait pid
-
     q.close
+
     q.join
 
     assert_equal reps, res.length
@@ -67,6 +69,7 @@ class TestWorkQueue < Test::Unit::TestCase
     reps = 10_000
     q = WorkQueue.new num do |obj|
       [Process.pid.to_s, obj.to_s] * " "
+      :ignore
     end
 
     q.ignore_ouput
@@ -76,14 +79,10 @@ class TestWorkQueue < Test::Unit::TestCase
       res << out
     end
 
-    pid = Process.fork do
-      reps.times do |i|
-        q.write i
-      end
+    reps.times do |i|
+      q.write i
     end
 
-    Process.wait pid
-
     q.close
     q.join
 
@@ -91,11 +90,12 @@ class TestWorkQueue < Test::Unit::TestCase
   end
 
   def test_queue_error
-    num = 10
+    num = 100
     reps = 10_000
 
+    sss 0
     q = WorkQueue.new num do |obj|
-      raise ScoutException if rand < 1
+      raise ScoutException if rand < 0.1
       [Process.pid.to_s, obj.to_s] * " "
     end
 
@@ -110,11 +110,39 @@ class TestWorkQueue < Test::Unit::TestCase
       end
     end
 
-    Process.wait pid
+    Process.waitpid pid
+    q.close
 
     assert_raise ScoutException do
+      q.join
+      t.join
+    end
+  end
+
+  def test_queue_error_in_input
+    num = 100
+    reps = 10_000
+
+    q = WorkQueue.new num do |obj|
+      [Process.pid.to_s, obj.to_s] * " "
+    end
+
+    res = []
+    q.process do |out|
+      raise ScoutException if rand < 0.01
+      res << out
+    end
+
+    pid = Process.fork do
+      reps.times do |i|
+        q.write i
+      end
       q.close
+    end
+
+    assert_raise ScoutException do
       q.join
+      t.join
     end
   end
 end