scout-gear 7.2.0 → 7.3.0

data/test/scout/work_queue/test_worker.rb

@@ -16,27 +16,84 @@ class TestQueueWorker < Test::Unit::TestCase
     end
   end
 
-  def test_semaphore
-    ScoutSemaphore.with_semaphore 1 do |sem|
+  def test_semaphore_pipe
+
+    2.times do
+      num_lines = 10
+      num_workers = 100
 
       TmpFile.with_file do |outfile|
-        2.times do
+        Open.rm(outfile)
+        ScoutSemaphore.with_semaphore 1 do |sem|
           sout = Open.open_pipe do |sin|
-            workers = 100.times.collect{ WorkQueue::Worker.new }
+            workers = num_workers.times.collect{ WorkQueue::Worker.new }
             workers.each do |w|
               w.run do
                 ScoutSemaphore.synchronize(sem) do
-                  10.times do
-                    sin.puts Process.pid
+                  sin.puts "Start - #{Process.pid}"
+                  num_lines.times do |i|
+                    sin.puts "line-#{i}-#{Process.pid}"
                   end
+                  sin.puts "End - #{Process.pid}"
                 end
               end
             end
+            sin.close
 
             WorkQueue::Worker.join(workers)
           end
+
           Open.consume_stream(sout, false, outfile)
+          txt = Open.read(outfile)
+          pid_list = txt.split("\n")
+          
+          assert_equal (num_lines + 2) * num_workers, pid_list.length
+
+          assert_nothing_raised do
+            seen = []
+            current = nil
+            pid_list.each do |pid|
+              if pid != current
+                raise "Out of order #{Log.fingerprint seen} #{ pid }" if seen.include? pid
+              end
+              current = pid
+              seen << pid
+            end
+          end
+        end
+      end
+    end
+  end
+  def test_semaphore
+
+    2.times do
+      num_lines = 10
+      num_workers = 500
+
+      TmpFile.with_file do |outfile|
+        Open.rm(outfile)
+        ScoutSemaphore.with_semaphore 1 do |sem|
+          workers = num_workers.times.collect{ WorkQueue::Worker.new }
+          Open.touch(outfile)
+          workers.each do |w|
+            w.run do
+              ScoutSemaphore.synchronize(sem) do
+                sin = Open.open(outfile, :mode => 'a')
+                sin.puts "Start - #{Process.pid}"
+                  num_lines.times do |i|
+                    sin.puts "line-#{i}-#{Process.pid}"
+                  end
+                sin.puts "End - #{Process.pid}"
+                sin.close
+              end
+            end
+          end
+
+          WorkQueue::Worker.join(workers)
+
+
           pid_list = Open.read(outfile).split("\n")
+          assert_equal (num_lines + 2) * num_workers, pid_list.length
 
           assert_nothing_raised do
             seen = []

data/test/scout/workflow/step/test_load.rb

@@ -41,7 +41,7 @@ class TestStepLoad < Test::Unit::TestCase
       end
     end
 
-    step2 = wf.job(:step2, :input1 => "TEST")
+    step2 = wf.job(:step2, nil, :input1 => "TEST_STRING")
     step1 = step2.step(:step1)
 
     step2.run
@@ -53,8 +53,8 @@ class TestStepLoad < Test::Unit::TestCase
         Open.mv step1.info_file, dir.var.jobs.RelocateWorkflow.step1[File.basename(step1.info_file)]
 
         new_step2 = Step.load(step2.path)
-        assert_equal "TEST".reverse, new_step2.load
-        assert_equal "TEST", new_step2.dependencies.first.load
+        assert_equal "TEST_STRING".reverse, new_step2.load
+        assert_equal "TEST_STRING", new_step2.dependencies.first.load
       end
     end
 

data/test/scout/workflow/test_step.rb

@@ -27,7 +27,6 @@ class TestWorkflowStep < Test::Unit::TestCase
 
     step2.dependencies = [step1]
 
-
     assert_equal "12 has 2 characters", step2.run
     assert_equal "12 has 2 characters", step2.run
   end
@@ -50,7 +49,7 @@ class TestWorkflowStep < Test::Unit::TestCase
 
     step2 = Step.new tmpfile.step2 do 
       step1 = dependencies.first
-      stream = step1.get_stream
+      stream = step1.stream
 
       Open.open_pipe do |sin|
         while line = stream.gets
@@ -87,7 +86,7 @@ class TestWorkflowStep < Test::Unit::TestCase
 
     assert step1.path.read.end_with? "line-#{times-1}\n"
     assert_equal times/2, lines.length
-    assert_equal times/2, step2.path.read.split("\n").length
+    assert_equal times/2, step2.join.path.read.split("\n").length
   end
 
   def test_streaming_duplicate
@@ -108,7 +107,7 @@ class TestWorkflowStep < Test::Unit::TestCase
 
     step2 = Step.new tmpfile.step2 do 
       step1 = dependencies.first
-      stream = step1.get_stream
+      stream = step1.stream
 
       Open.open_pipe do |sin|
         while line = stream.gets
@@ -123,7 +122,7 @@ class TestWorkflowStep < Test::Unit::TestCase
 
     step3 = Step.new tmpfile.step3 do 
       step1, step2 = dependencies
-      stream = step2.get_stream
+      stream = step2.stream
 
       Open.open_pipe do |sin|
         while line = stream.gets
@@ -164,7 +163,7 @@ class TestWorkflowStep < Test::Unit::TestCase
 
     step2 = Step.new tmpfile.step2 do 
       step1 = dependencies.first
-      stream = step1.get_stream
+      stream = step1.stream
 
       Open.open_pipe do |sin|
         while line = stream.gets
@@ -179,7 +178,7 @@ class TestWorkflowStep < Test::Unit::TestCase
 
     step3 = Step.new tmpfile.step3 do 
       step1 = dependencies.first
-      stream = step1.get_stream
+      stream = step1.stream
 
       Open.open_pipe do |sin|
         while line = stream.gets
@@ -199,14 +198,14 @@ class TestWorkflowStep < Test::Unit::TestCase
       mutex = Mutex.new
       Open.open_pipe do |sin|
         t2 = Thread.new do
-          stream2 = step2.get_stream
+          stream2 = step2.stream
           while line = stream2.gets
             sin.puts line
           end
         end
 
         t3 = Thread.new do
-          stream3 = step3.get_stream
+          stream3 = step3.stream
           while line = stream3.gets
             sin.puts line
           end
@@ -220,12 +219,13 @@ class TestWorkflowStep < Test::Unit::TestCase
 
     lines = []
     io = step4.run
-    Log::ProgressBar.with_bar do |b|
+    Log::ProgressBar.with_bar severity: 0 do |b|
       while line = io.gets
         b.tick
         lines << line.strip
       end
     end
+    io.close
 
     assert_equal times, lines.length
   end

data/test/test_helper.rb

@@ -33,12 +33,14 @@ class Test::Unit::TestCase
   end
 
   def tmpdir
-    @tmpdir = Path.setup('tmp/test_tmpdir').find
+    @tmpdir ||= Path.setup('tmp/test_tmpdir').find
   end
 
   setup do
+    TmpFile.tmpdir = tmpdir.tmpfiles
     Log::ProgressBar.default_severity = 0
     Persist.cache_dir = tmpdir.var.cache
+    Persist::MEMORY_CACHE.clear
     Open.remote_cache_dir = tmpdir.var.cache
     Workflow.directory = tmpdir.var.jobs
     Workflow.workflows.each{|wf| wf.directory = Workflow.directory[wf.name] }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scout-gear
 version: !ruby/object:Gem::Version
-  version: 7.2.0
+  version: 7.3.0
 platform: ruby
 authors:
 - Miguel Vazquez
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-05-09 00:00:00.000000000 Z
+date: 2023-05-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: term-ansicolor
@@ -119,6 +119,7 @@ files:
 - lib/scout/misc/digest.rb
 - lib/scout/misc/filesystem.rb
 - lib/scout/misc/format.rb
+- lib/scout/misc/helper.rb
 - lib/scout/misc/insist.rb
 - lib/scout/misc/monitor.rb
 - lib/scout/misc/system.rb
@@ -154,13 +155,19 @@ files:
 - lib/scout/tsv.rb
 - lib/scout/tsv/dumper.rb
 - lib/scout/tsv/index.rb
+- lib/scout/tsv/open.rb
 - lib/scout/tsv/parser.rb
 - lib/scout/tsv/path.rb
 - lib/scout/tsv/persist.rb
 - lib/scout/tsv/persist/adapter.rb
+- lib/scout/tsv/persist/fix_width_table.rb
+- lib/scout/tsv/persist/serialize.rb
 - lib/scout/tsv/persist/tokyocabinet.rb
 - lib/scout/tsv/traverse.rb
 - lib/scout/tsv/util.rb
+- lib/scout/tsv/util/filter.rb
+- lib/scout/tsv/util/process.rb
+- lib/scout/tsv/util/select.rb
 - lib/scout/work_queue.rb
 - lib/scout/work_queue/socket.rb
 - lib/scout/work_queue/worker.rb
@@ -173,6 +180,7 @@ files:
 - lib/scout/workflow/step/file.rb
 - lib/scout/workflow/step/info.rb
 - lib/scout/workflow/step/load.rb
+- lib/scout/workflow/step/progress.rb
 - lib/scout/workflow/step/provenance.rb
 - lib/scout/workflow/task.rb
 - lib/scout/workflow/task/inputs.rb
@@ -234,13 +242,18 @@ files:
 - test/scout/test_work_queue.rb
 - test/scout/test_workflow.rb
 - test/scout/tsv/persist/test_adapter.rb
+- test/scout/tsv/persist/test_fix_width_table.rb
 - test/scout/tsv/persist/test_tokyocabinet.rb
 - test/scout/tsv/test_dumper.rb
 - test/scout/tsv/test_index.rb
+- test/scout/tsv/test_open.rb
 - test/scout/tsv/test_parser.rb
 - test/scout/tsv/test_persist.rb
 - test/scout/tsv/test_traverse.rb
 - test/scout/tsv/test_util.rb
+- test/scout/tsv/util/test_filter.rb
+- test/scout/tsv/util/test_process.rb
+- test/scout/tsv/util/test_select.rb
 - test/scout/work_queue/test_socket.rb
 - test/scout/work_queue/test_worker.rb
 - test/scout/workflow/step/test_dependencies.rb
@@ -260,7 +273,7 @@ homepage: http://github.com/mikisvaz/scout-gear
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -275,8 +288,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.11
-signing_key: 
+rubygems_version: 3.2.15
+signing_key:
 specification_version: 4
 summary: basic gear for scouts
 test_files: []