scout-gear 7.1.0 → 7.3.0

data/test/scout/workflow/test_step.rb

@@ -15,22 +15,218 @@ class TestWorkflowStep < Test::Unit::TestCase
   end
 
   def test_dependency
-    Log.with_severity 0 do
-      tmpfile = tmpdir.test_step
-      step1 = Step.new tmpfile.step1, ["12"] do |s|
-        s.length
+    tmpfile = tmpdir.test_step
+    step1 = Step.new tmpfile.step1, ["12"] do |s|
+      s.length
+    end
+
+    step2 = Step.new tmpfile.step2 do 
+      step1 = dependencies.first
+      step1.inputs.first + " has " + step1.load.to_s + " characters"
+    end
+
+    step2.dependencies = [step1]
+
+    assert_equal "12 has 2 characters", step2.run
+    assert_equal "12 has 2 characters", step2.run
+  end
+
+  def test_streaming
+    tmpfile = tmpdir.test_step
+
+    times = 10_000
+    sleep = 1 / times
+
+    step1 = Step.new tmpfile.step1, [times, sleep] do |times,sleep|
+      Open.open_pipe do |sin|
+        times.times do |i|
+          sin.puts "line-#{i}"
+          sleep sleep
+        end
+      end
+    end
+    step1.type = :array
+
+    step2 = Step.new tmpfile.step2 do 
+      step1 = dependencies.first
+      stream = step1.stream
+
+      Open.open_pipe do |sin|
+        while line = stream.gets
+          num = line.split("-").last
+          next if num.to_i % 2 == 1
+          sin.puts line
+        end
+      end
+    end
+    step2.type = :array
+    step2.dependencies = [step1]
+
+    step1.run
+    step1.join
+    assert step1.path.read.end_with? "line-#{times-1}\n"
+
+    step1.clean
+
+    stream = step2.exec
+    lines = []
+    while line = stream.gets
+      lines << line
+    end
+    assert_equal times/2, lines.length
+
+    stream = step2.run
+    assert step1.streaming?
+    assert step2.streaming?
+
+    lines = []
+    while line = stream.gets
+      lines << line
+    end
+
+    assert step1.path.read.end_with? "line-#{times-1}\n"
+    assert_equal times/2, lines.length
+    assert_equal times/2, step2.join.path.read.split("\n").length
+  end
+
+  def test_streaming_duplicate
+    tmpfile = tmpdir.test_step
+
+    times = 10_000
+    sleep = 0.01 / times
+
+    step1 = Step.new tmpfile.step1, [times, sleep] do |times,sleep|
+      Open.open_pipe do |sin|
+        times.times do |i|
+          sin.puts "line-#{i}"
+          sleep sleep
+        end
+      end
+    end
+    step1.type = :array
+
+    step2 = Step.new tmpfile.step2 do 
+      step1 = dependencies.first
+      stream = step1.stream
+
+      Open.open_pipe do |sin|
+        while line = stream.gets
+          num = line.split("-").last
+          next if num.to_i % 2 == 1
+          sin.puts line
+        end
       end
+    end
+    step2.type = :array
+    step2.dependencies = [step1]
 
-      step2 = Step.new tmpfile.step2 do 
-        step1 = dependencies.first
-        step1.inputs.first + " has " + step1.load.to_s + " characters"
+    step3 = Step.new tmpfile.step3 do 
+      step1, step2 = dependencies
+      stream = step2.stream
+
+      Open.open_pipe do |sin|
+        while line = stream.gets
+          num = line.split("-").last
+          next if num.to_i % 2 == 1
+          sin.puts line
+        end
       end
+    end
+    step3.type = :array
+    step3.dependencies = [step1, step2]
 
-      step2.dependencies = [step1]
+    step3.recursive_clean
 
+    stream = step3.run
+    out = []
+    while l = stream.gets
+      out << l
+    end
+    assert_equal times/2, out.length
+  end
 
-      assert_equal "12 has 2 characters", step2.run
-      assert_equal "12 has 2 characters", step2.run
+  def test_fork_stream
+    tmpfile = tmpdir.test_step
+
+    times = 10_000
+    sleep = 0.1 / times
+
+    step1 = Step.new tmpfile.step1, [times, sleep] do |times,sleep|
+      Open.open_pipe do |sin|
+        times.times do |i|
+          sin.puts "line-#{i}"
+          sleep sleep
+        end
+      end
+    end
+    step1.type = :array
+
+    step2 = Step.new tmpfile.step2 do 
+      step1 = dependencies.first
+      stream = step1.stream
+
+      Open.open_pipe do |sin|
+        while line = stream.gets
+          num = line.split("-").last
+          next if num.to_i % 2 == 1
+          sin.puts "S2: " + line
+        end
+      end
     end
+    step2.type = :array
+    step2.dependencies = [step1]
+
+    step3 = Step.new tmpfile.step3 do 
+      step1 = dependencies.first
+      stream = step1.stream
+
+      Open.open_pipe do |sin|
+        while line = stream.gets
+          num = line.split("-").last
+          next if num.to_i % 2 == 0
+          sin.puts "S3: " + line
+        end
+      end
+    end
+    step3.type = :array
+    step3.dependencies = [step1]
+
+
+    step4 = Step.new tmpfile.step4 do 
+      step2, step3 = dependencies
+
+      mutex = Mutex.new
+      Open.open_pipe do |sin|
+        t2 = Thread.new do
+          stream2 = step2.stream
+          while line = stream2.gets
+            sin.puts line
+          end
+        end
+
+        t3 = Thread.new do
+          stream3 = step3.stream
+          while line = stream3.gets
+            sin.puts line
+          end
+        end
+        t2.join
+        t3.join
+      end
+    end
+    step4.type = :array
+    step4.dependencies = [step2, step3]
+
+    lines = []
+    io = step4.run
+    Log::ProgressBar.with_bar severity: 0 do |b|
+      while line = io.gets
+        b.tick
+        lines << line.strip
+      end
+    end
+    io.close
+
+    assert_equal times, lines.length
   end
 end

data/test/scout/workflow/test_task.rb

@@ -37,7 +37,6 @@ class TestTask < Test::Unit::TestCase
   end
 
   def test_override_inputs_block
-    Log.severity = 0
     wf = Workflow.annonymous_workflow "TaskInputs" do
       input :input1, :string
       task :step1 => :string do |i| i end
@@ -55,7 +54,6 @@ class TestTask < Test::Unit::TestCase
   end
 
   def test_task_override_dep
-    Log.severity = 0
     wf = Workflow.annonymous_workflow "TaskInputs" do
       input :input1, :integer
       task :step1 => :integer do |i| i end
@@ -171,7 +169,6 @@ class TestTask < Test::Unit::TestCase
       end
     end
 
-    Log.severity = 0
     job = wf.job(:my_sum, :vv1 => 2, :vv2 => 3)
     assert_equal 5, job.run
   end

data/test/test_helper.rb

@@ -22,17 +22,25 @@ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 #require 'scout/helper/misc/development'
 require 'scout/tmpfile'
 require 'scout/log'
+require 'scout/open'
 require 'scout/persist'
 require 'scout/workflow'
 
 class Test::Unit::TestCase
 
+  def assert_equal_path(path1, path2)
+    assert_equal File.expand_path(path1), File.expand_path(path2)
+  end
+
   def tmpdir
-    @tmpdir = Path.setup('tmp/test_tmpdir').find
+    @tmpdir ||= Path.setup('tmp/test_tmpdir').find
   end
 
   setup do
+    TmpFile.tmpdir = tmpdir.tmpfiles
+    Log::ProgressBar.default_severity = 0
     Persist.cache_dir = tmpdir.var.cache
+    Persist::MEMORY_CACHE.clear
     Open.remote_cache_dir = tmpdir.var.cache
     Workflow.directory = tmpdir.var.jobs
     Workflow.workflows.each{|wf| wf.directory = Workflow.directory[wf.name] }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scout-gear
 version: !ruby/object:Gem::Version
-  version: 7.1.0
+  version: 7.3.0
 platform: ruby
 authors:
 - Miguel Vazquez
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-05-01 00:00:00.000000000 Z
+date: 2023-05-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: term-ansicolor
@@ -97,10 +97,12 @@ files:
 - Rakefile
 - VERSION
 - bin/scout
+- lib/rbbt-scout.rb
 - lib/scout-gear.rb
 - lib/scout.rb
 - lib/scout/cmd.rb
 - lib/scout/concurrent_stream.rb
+- lib/scout/config.rb
 - lib/scout/exceptions.rb
 - lib/scout/indiferent_hash.rb
 - lib/scout/indiferent_hash/case_insensitive.rb
@@ -117,8 +119,11 @@ files:
 - lib/scout/misc/digest.rb
 - lib/scout/misc/filesystem.rb
 - lib/scout/misc/format.rb
+- lib/scout/misc/helper.rb
 - lib/scout/misc/insist.rb
 - lib/scout/misc/monitor.rb
+- lib/scout/misc/system.rb
+- lib/scout/named_array.rb
 - lib/scout/open.rb
 - lib/scout/open/lock.rb
 - lib/scout/open/remote.rb
@@ -137,6 +142,7 @@ files:
 - lib/scout/resource/produce.rb
 - lib/scout/resource/produce/rake.rb
 - lib/scout/resource/scout.rb
+- lib/scout/resource/software.rb
 - lib/scout/resource/util.rb
 - lib/scout/semaphore.rb
 - lib/scout/simple_opt.rb
@@ -147,7 +153,21 @@ files:
 - lib/scout/simple_opt/setup.rb
 - lib/scout/tmpfile.rb
 - lib/scout/tsv.rb
+- lib/scout/tsv/dumper.rb
+- lib/scout/tsv/index.rb
+- lib/scout/tsv/open.rb
 - lib/scout/tsv/parser.rb
+- lib/scout/tsv/path.rb
+- lib/scout/tsv/persist.rb
+- lib/scout/tsv/persist/adapter.rb
+- lib/scout/tsv/persist/fix_width_table.rb
+- lib/scout/tsv/persist/serialize.rb
+- lib/scout/tsv/persist/tokyocabinet.rb
+- lib/scout/tsv/traverse.rb
+- lib/scout/tsv/util.rb
+- lib/scout/tsv/util/filter.rb
+- lib/scout/tsv/util/process.rb
+- lib/scout/tsv/util/select.rb
 - lib/scout/work_queue.rb
 - lib/scout/work_queue/socket.rb
 - lib/scout/work_queue/worker.rb
@@ -155,8 +175,13 @@ files:
 - lib/scout/workflow/definition.rb
 - lib/scout/workflow/documentation.rb
 - lib/scout/workflow/step.rb
+- lib/scout/workflow/step/config.rb
+- lib/scout/workflow/step/dependencies.rb
+- lib/scout/workflow/step/file.rb
 - lib/scout/workflow/step/info.rb
 - lib/scout/workflow/step/load.rb
+- lib/scout/workflow/step/progress.rb
+- lib/scout/workflow/step/provenance.rb
 - lib/scout/workflow/task.rb
 - lib/scout/workflow/task/inputs.rb
 - lib/scout/workflow/usage.rb
@@ -173,6 +198,7 @@ files:
 - scout_commands/workflow/task_old
 - share/color/color_names
 - share/color/diverging_colors.hex
+- share/software/install_helpers
 - test/scout/indiferent_hash/test_case_insensitive.rb
 - test/scout/indiferent_hash/test_options.rb
 - test/scout/log/test_color.rb
@@ -180,6 +206,7 @@ files:
 - test/scout/misc/test_digest.rb
 - test/scout/misc/test_filesystem.rb
 - test/scout/misc/test_insist.rb
+- test/scout/misc/test_system.rb
 - test/scout/open/test_lock.rb
 - test/scout/open/test_remote.rb
 - test/scout/open/test_stream.rb
@@ -191,6 +218,7 @@ files:
 - test/scout/persist/test_serialize.rb
 - test/scout/resource/test_path.rb
 - test/scout/resource/test_produce.rb
+- test/scout/resource/test_software.rb
 - test/scout/resource/test_util.rb
 - test/scout/simple_opt/test_doc.rb
 - test/scout/simple_opt/test_get.rb
@@ -198,10 +226,12 @@ files:
 - test/scout/simple_opt/test_setup.rb
 - test/scout/test_cmd.rb
 - test/scout/test_concurrent_stream.rb
+- test/scout/test_config.rb
 - test/scout/test_indiferent_hash.rb
 - test/scout/test_log.rb
 - test/scout/test_meta_extension.rb
 - test/scout/test_misc.rb
+- test/scout/test_named_array.rb
 - test/scout/test_open.rb
 - test/scout/test_path.rb
 - test/scout/test_persist.rb
@@ -211,11 +241,25 @@ files:
 - test/scout/test_tsv.rb
 - test/scout/test_work_queue.rb
 - test/scout/test_workflow.rb
+- test/scout/tsv/persist/test_adapter.rb
+- test/scout/tsv/persist/test_fix_width_table.rb
+- test/scout/tsv/persist/test_tokyocabinet.rb
+- test/scout/tsv/test_dumper.rb
+- test/scout/tsv/test_index.rb
+- test/scout/tsv/test_open.rb
 - test/scout/tsv/test_parser.rb
+- test/scout/tsv/test_persist.rb
+- test/scout/tsv/test_traverse.rb
+- test/scout/tsv/test_util.rb
+- test/scout/tsv/util/test_filter.rb
+- test/scout/tsv/util/test_process.rb
+- test/scout/tsv/util/test_select.rb
 - test/scout/work_queue/test_socket.rb
 - test/scout/work_queue/test_worker.rb
+- test/scout/workflow/step/test_dependencies.rb
 - test/scout/workflow/step/test_info.rb
 - test/scout/workflow/step/test_load.rb
+- test/scout/workflow/step/test_provenance.rb
 - test/scout/workflow/task/test_inputs.rb
 - test/scout/workflow/test_definition.rb
 - test/scout/workflow/test_documentation.rb
@@ -229,7 +273,7 @@ homepage: http://github.com/mikisvaz/scout-gear
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -244,8 +288,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.11
-signing_key: 
+rubygems_version: 3.2.15
+signing_key:
 specification_version: 4
 summary: basic gear for scouts
 test_files: []