scout-gear 5.2.0 → 7.1.0

data/test/scout/tsv/test_parser.rb

@@ -0,0 +1,87 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+class TestTSVParser < Test::Unit::TestCase
+
+  def test_parse_line
+    line = (0..10).to_a * "\t"
+    key, values = TSV.parse_line(line)
+
+    assert_equal "0", key
+    assert_equal (1..10).collect{|v| v.to_s }, values
+  end
+
+  def test_parse_double
+    line = (0..10).collect{|v| v == 0 ? v : [v,v] * "|" } * "\t"
+    key, values = TSV.parse_line(line, type: :double, cast: :to_i)
+
+    assert_equal "0", key
+    assert_equal (1..10).collect{|v| [v,v] }, values
+  end
+
+  def __test_benchmark
+    num = 10_000
+    txt = num.times.inject(nil) do |acc,i|
+      (acc.nil? ? "" : acc << "\n") << (0..10).collect{|v| v == 0 ? i : [v,v] * "|" } * "\t"
+    end 
+
+    txt = StringIO.new(([txt] * (10))*"\n")
+    Misc.benchmark 1 do
+    #Misc.profile do
+      data = TSV.parse_stream(txt, fix: true, type: :double, bar: true, merge: :concat)
+      assert_equal num, data.size
+      assert_equal 20, data['1'][0].length
+    end
+  end
+
+  def test_parse_stream
+    lines =<<-EOF
+1 2 3 4 5
+11 12 13 14 15
+    EOF
+
+    lines = StringIO.new lines
+
+    data = TSV.parse_stream lines, sep: " "
+    assert_equal data["1"], %w(2 3 4 5)
+  end
+
+  def test_parse_stream_block
+    lines =<<-EOF
+1 2 3 4 5
+11 12 13 14 15
+    EOF
+
+    lines = StringIO.new lines
+
+    sum = 0
+    res = TSV.parse_stream(lines, sep: " ") do |k,values|
+      sum += values.inject(0){|acc,i| acc += i.to_i }
+    end
+    assert_equal 68, sum
+  end
+
+  def test_parse_header
+    header =<<-EOF
+#: :sep=" "
+#Key ValueA ValueB
+k A B
+    EOF
+    header = StringIO.new header
+
+    assert_equal "Key", TSV.parse_header(header)[1]
+  end
+
+  def test_parse
+    header =<<-EOF
+#: :sep=" "#:type=:double
+#Key ValueA ValueB
+k a|A b|B
+    EOF
+    header = StringIO.new header
+
+    tsv = TSV.parse(header)
+    assert_equal 'a', tsv['k'][0][0]
+  end
+end

data/test/scout/work_queue/test_socket.rb

@@ -0,0 +1,46 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestSocket < Test::Unit::TestCase
+  def test_simple
+    socket = WorkQueue::Socket.new 
+
+    socket.write 1
+    socket.write 2
+    socket.write "STRING"
+    socket.write :string
+
+    assert_equal 1, socket.read
+    assert_equal 2, socket.read
+    assert_equal "STRING", socket.read
+    assert_equal :string, socket.read
+
+    socket.close_write
+    assert_raise ClosedStream do
+      socket.read
+    end
+  end
+
+  def __test_speed
+    sss 0
+    socket = WorkQueue::Socket.new 
+
+    num = 10_000
+
+    Thread.new do
+      num.times do |i|
+        socket.write nil
+      end
+      socket.write DoneProcessing.new 
+    end
+
+    bar = Log::ProgressBar.new num
+    while true
+      i = socket.read
+      bar.tick
+      break if DoneProcessing === i
+    end
+    bar.done
+  end
+end
+

data/test/scout/work_queue/test_worker.rb

@@ -0,0 +1,147 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/semaphore'
+require 'scout/work_queue/socket'
+class TestQueueWorker < Test::Unit::TestCase
+  def test_simple
+    worker = WorkQueue::Worker.new
+    TmpFile.with_file do |file|
+      worker.run do
+        Open.write file, "TEST"
+      end
+      worker.join
+
+      assert_equal "TEST", Open.read(file)
+    end
+  end
+
+  def test_semaphore
+    ScoutSemaphore.with_semaphore 1 do |sem|
+
+      TmpFile.with_file do |outfile|
+        2.times do
+          sout = Open.open_pipe do |sin|
+            workers = 100.times.collect{ WorkQueue::Worker.new }
+            workers.each do |w|
+              w.run do
+                ScoutSemaphore.synchronize(sem) do
+                  10.times do
+                    sin.puts Process.pid
+                  end
+                end
+              end
+            end
+
+            WorkQueue::Worker.join(workers)
+          end
+          Open.consume_stream(sout, false, outfile)
+          pid_list = Open.read(outfile).split("\n")
+
+          assert_nothing_raised do
+            seen = []
+            current = nil
+            pid_list.each do |pid|
+              if pid != current
+                raise "Out of order" if seen.include? pid
+              end
+              current = pid
+              seen << pid
+            end
+          end
+        end
+      end
+    end
+  end
+
+  def test_process
+    input = WorkQueue::Socket.new
+    output = WorkQueue::Socket.new
+
+    workers = 10.times.collect{ WorkQueue::Worker.new }
+    workers.each do |w|
+      w.process(input, output) do |obj|
+        [Process.pid, obj.inspect] * " "
+      end
+    end
+
+    read = Thread.new do 
+      begin
+        while obj = output.read
+          if DoneProcessing === obj
+            pid = obj.pid
+            workers.delete_if{|w| w.pid = pid }
+            break if workers.empty?
+          end
+        end
+      end
+    end
+
+    write = Thread.new do
+      100.times do |i|
+        input.write i
+      end
+      10.times do
+        input.write DoneProcessing.new
+      end
+      input.close_write
+    end
+
+    write.join
+    read.join
+
+    WorkQueue::Worker.join workers
+    input.clean
+    output.clean
+  end
+
+  def test_process_exception
+    input = WorkQueue::Socket.new
+    output = WorkQueue::Socket.new
+
+    workers = 10.times.collect{ WorkQueue::Worker.new }
+    workers.each do |w|
+      w.process(input, output) do |obj|
+        raise ScoutException
+        [Process.pid, obj.inspect] * " "
+      end
+    end
+
+    read = Thread.new do 
+      Thread.current.report_on_exception = false
+      begin
+        while obj = output.read
+          if DoneProcessing === obj
+            pid = obj.pid
+            workers.delete_if{|w| w.pid = pid }
+            break if workers.empty?
+          end
+          raise obj if Exception === obj
+        end
+      end
+    end
+
+    write = Thread.new do
+      Thread.report_on_exception = false
+      100.times do |i|
+        input.write i
+      end
+      10.times do
+        input.write DoneProcessing.new
+      end
+      input.close_write
+    end
+
+    write.join
+
+    assert_raise ScoutException do
+      read.join
+    end
+
+    WorkQueue::Worker.join workers
+    input.clean
+    output.clean
+  end
+
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scout-gear
 version: !ruby/object:Gem::Version
-  version: 5.2.0
+  version: 7.1.0
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-04-28 00:00:00.000000000 Z
+date: 2023-05-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: term-ansicolor
@@ -138,6 +138,7 @@ files:
 - lib/scout/resource/produce/rake.rb
 - lib/scout/resource/scout.rb
 - lib/scout/resource/util.rb
+- lib/scout/semaphore.rb
 - lib/scout/simple_opt.rb
 - lib/scout/simple_opt/accessor.rb
 - lib/scout/simple_opt/doc.rb
@@ -145,6 +146,11 @@ files:
 - lib/scout/simple_opt/parse.rb
 - lib/scout/simple_opt/setup.rb
 - lib/scout/tmpfile.rb
+- lib/scout/tsv.rb
+- lib/scout/tsv/parser.rb
+- lib/scout/work_queue.rb
+- lib/scout/work_queue/socket.rb
+- lib/scout/work_queue/worker.rb
 - lib/scout/workflow.rb
 - lib/scout/workflow/definition.rb
 - lib/scout/workflow/documentation.rb
@@ -165,8 +171,11 @@ files:
 - scout_commands/workflow/list
 - scout_commands/workflow/task
 - scout_commands/workflow/task_old
+- share/color/color_names
+- share/color/diverging_colors.hex
 - test/scout/indiferent_hash/test_case_insensitive.rb
 - test/scout/indiferent_hash/test_options.rb
+- test/scout/log/test_color.rb
 - test/scout/log/test_progress.rb
 - test/scout/misc/test_digest.rb
 - test/scout/misc/test_filesystem.rb
@@ -197,8 +206,14 @@ files:
 - test/scout/test_path.rb
 - test/scout/test_persist.rb
 - test/scout/test_resource.rb
+- test/scout/test_semaphore.rb
 - test/scout/test_tmpfile.rb
+- test/scout/test_tsv.rb
+- test/scout/test_work_queue.rb
 - test/scout/test_workflow.rb
+- test/scout/tsv/test_parser.rb
+- test/scout/work_queue/test_socket.rb
+- test/scout/work_queue/test_worker.rb
 - test/scout/workflow/step/test_info.rb
 - test/scout/workflow/step/test_load.rb
 - test/scout/workflow/task/test_inputs.rb