scout-gear 7.3.0 → 8.1.0

data/test/scout/workflow/task/test_dependencies.rb

@@ -0,0 +1,355 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/workflow'
+class TestTaskDependencies < Test::Unit::TestCase
+  def test_task_override_dep_exec
+    wf = Workflow.annonymous_workflow "TaskInputs" do
+      input :input1, :integer
+      task :step1 => :integer do |i| i end
+
+      dep :step1
+      input :input2, :integer, "Integer", 3
+      task :step2 => :integer do |i| i * step(:step1).load end
+    end
+
+    assert_equal 6, wf.job(:step2, :input1 => 2, :input2 => 3).exec
+
+    step1_job = wf.job(:step1, :input1 => 6)
+    assert_equal 18, wf.job(:step2, :input1 => 2, "TaskInputs#step1" => step1_job).exec
+
+    assert_equal 18, wf.job(:step2, :input1 => 2, "TaskInputs#step1" => step1_job).exec
+
+    assert_equal 18, wf.job(:step2, "TaskInputs#step1" => step1_job).exec
+  end
+
+  def test_task_override_dep
+    wf = Workflow.annonymous_workflow "TaskInputs" do
+      input :input1, :integer
+      task :step1 => :integer do |i| i end
+
+      dep :step1
+      input :input2, :integer, "Integer", 3
+      task :step2 => :integer do |i| i * step(:step1).load end
+    end
+
+    assert_equal 6, wf.job(:step2, :input1 => 2, :input2 => 3).run
+
+    step1_job = wf.job(:step1, :input1 => 6)
+    assert_equal 18, wf.job(:step2, :input1 => 2, "TaskInputs#step1" => step1_job).run
+
+    assert_equal 18, wf.job(:step2, :input1 => 2, "TaskInputs#step1" => step1_job).run
+
+    assert_equal 18, wf.job(:step2, "TaskInputs#step1" => step1_job).run
+  end
+
+  def test_input_dep
+    wf = Workflow.annonymous_workflow "TaskInputs" do
+      input :v1, :integer
+      input :v2, :integer
+      task :sum => :integer do |v1,v2|
+        v1 + v2
+      end
+
+      input :input1, :integer
+      task :step1 => :integer do |i| i end
+
+      input :input2, :integer
+      task :step2 => :integer do |i| i end
+
+      dep :step1, :input1 => 2
+      dep :step2, :input2 => 3
+      dep :sum, :v1 => :step1, :v2 => :step2
+      task :my_sum => :integer do
+        dependencies.last.load
+      end
+
+      dep :my_sum
+      task :double => :integer do
+        step(:my_sum).load * 2 
+      end
+    end
+
+    job = wf.job(:my_sum)
+    assert_equal 5, job.run
+    assert_equal Task::DEFAULT_NAME, job.name
+
+    job = wf.job(:double)
+    assert_equal Task::DEFAULT_NAME, job.name
+
+    step1 = wf.job(:step1, :input1 => 3)
+    assert_equal 3, step1.run
+    job = wf.job(:double, "TaskInputs#step1" => step1)
+    assert_equal 12, job.run
+    assert_not_equal Task::DEFAULT_NAME, job.name
+
+    step1 = wf.job(:step1, :input1 => 4)
+    assert_equal 4, step1.run
+    job = wf.job(:double, "TaskInputs#step1" => step1)
+    assert_equal 14, job.run
+    assert_not_equal Task::DEFAULT_NAME, job.name
+  end
+
+  def test_input_dep_override
+    wf = Workflow.annonymous_workflow "TaskInputs" do
+      input :v1, :integer
+      input :v2, :integer
+      task :sum => :integer do |v1,v2|
+        v1 + v2
+      end
+
+      input :input1, :integer
+      task :step1 => :integer do |i| i end
+
+      input :input2, :integer
+      task :step2 => :integer do |i| i end
+
+      dep :step1
+      dep :step2
+      task :my_sum => :integer do
+        dependencies.inject(0){|acc,d| acc += d.load }
+      end
+    end
+
+    step2 = wf.job(:step2, :input2 => 4)
+    job = wf.job(:my_sum, :input1 => 2, :input2 => 3, "TaskInputs#step2"=> step2)
+    assert_equal 6, job.exec
+
+    job = wf.job(:my_sum, :input1 => 2, :input2 => 3)
+    assert_equal 5, job.run
+
+    TmpFile.with_file(4) do |file|
+      job = wf.job(:my_sum, :input1 => 2, :input2 => 3, "TaskInputs#step2"=> file)
+      assert_equal 6, job.run
+      assert_not_equal Task::DEFAULT_NAME, job.name
+    end
+  end
+
+  def test_input_rename
+    wf = Workflow.annonymous_workflow "TaskInputs" do
+      input :v1, :integer
+      input :v2, :integer
+      task :sum => :integer do |v1,v2|
+        v1 + v2
+      end
+
+      input :vv1, :integer
+      input :vv2, :integer
+      dep :sum, :v1 => :vv1, :v2 => :vv2
+      task :my_sum => :integer do
+        dependencies.last.load
+      end
+    end
+
+    job = wf.job(:my_sum, :vv1 => 2, :vv2 => 3)
+    assert_equal 5, job.run
+  end
+
+  def test_defaults_in_dep_block
+    wf = Workflow.annonymous_workflow "TaskInputs" do
+      input :v1, :integer
+      input :v2, :integer
+      task :sum => :integer do |v1,v2|
+        v1 + v2
+      end
+
+      input :vv1, :integer
+      input :vv2, :integer, nil, 3
+      dep :sum, :v1 => :placeholder, :v2 => :placeholder do |jobname,options,dependencies|
+        raise "Non-numeric value where integer expected" unless Numeric === options[:vv1]
+        {inputs: {v1: options[:vv1], v2: options[:vv2]} }
+      end
+      task :my_sum => :integer do
+        dependencies.last.load
+      end
+    end
+
+    job = wf.job(:my_sum, :vv1 => "2")
+    assert_equal 5, job.run
+  end
+
+  def test_dependency_jobname
+    wf = Workflow.annonymous_workflow "TaskInputs" do
+      input :v1, :integer
+      input :v2, :integer
+      task :sum => :integer do |v1,v2|
+        v1 + v2
+      end
+
+      input :vv1, :integer
+      input :vv2, :integer
+      dep :sum, :v1 => :vv1, :v2 => :vv2, :jobname => "OTHER_NAME"
+      task :my_sum => :integer do
+        dependencies.last.load
+      end
+    end
+
+    job = wf.job(:my_sum, "TEST_NAME", :vv1 => 2, :vv2 => 3)
+    assert_equal 5, job.run
+    assert_equal "TEST_NAME", job.clean_name
+    assert_equal "OTHER_NAME", job.step(:sum).clean_name
+  end
+
+  def test_no_param_last_job
+    wf = Workflow.annonymous_workflow "TaskInputs" do
+      input :v1, :integer
+      input :v2, :integer
+      task :sum => :integer do |v1,v2|
+        v1 + v2
+      end
+
+      dep :sum
+      task :my_sum => :integer do
+        dependencies.last.load
+      end
+    end
+
+    job = wf.job(:my_sum, :v1 => 2, :v2 => "3")
+    assert_equal 5, job.run
+    refute_equal Task::DEFAULT_NAME, job.name
+  end
+
+  def test_no_param_last_job_block
+    wf = Workflow.annonymous_workflow "TaskInputs" do
+      input :v1, :integer
+      input :v2, :integer
+      task :sum => :integer do |v1,v2|
+        v1 + v2
+      end
+
+      dep :sum do |jobname,options|
+        {inputs: options}
+      end
+      task :my_sum => :integer do
+        dependencies.last.load
+      end
+    end
+
+    job = wf.job(:my_sum, :v1 => 2, :v2 => "3")
+    assert_equal 5, job.run
+    refute_equal Task::DEFAULT_NAME, job.name
+  end
+
+
+  def test_override_inputs_block
+    wf = Workflow.annonymous_workflow "TaskInputs" do
+      input :input1, :string
+      task :step1 => :string do |i| i end
+
+      dep :step1, :input1 => 1  do |id,options|
+        {:inputs => options}
+      end
+      task :step2 => :string do |i| step(:step1).load end
+    end
+
+    job = wf.job(:step2, :input1 => 2)
+    assert_equal 1, job.run
+    assert_equal Task::DEFAULT_NAME, job.name
+    assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
+  end
+
+  def test_default_inputs_in_block
+    wf = Workflow.annonymous_workflow "TaskInputs" do
+      input :input1, :string
+      task :step1 => :string do |i| i end
+
+      dep :step1  do |id,options|
+        {}
+      end
+      task :step2 => :string do |i| step(:step1).load end
+    end
+
+    job = wf.job(:step2, "SOME_NAME", :input1 => 2)
+    assert_equal "SOME_NAME", job.step(:step1).clean_name
+    assert_equal 2, job.run
+  end
+
+  def test_override_inputs_block_array
+    wf = Workflow.annonymous_workflow "TaskInputs" do
+      input :input1, :string
+      task :step1 => :string do |i| i end
+
+      dep :step1, :input1 => 1  do |id,options|
+        [{:inputs => options}]
+      end
+      input :input2, :string
+      task :step2 => :string do |i| step(:step1).load end
+    end
+
+    job = wf.job(:step2, :input1 => 2)
+    assert_equal 1, job.run
+    assert_equal Task::DEFAULT_NAME, job.name
+    assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
+  end
+
+  def test_override_inputs
+    wf = Workflow.annonymous_workflow "TaskInputs" do
+      input :input1, :string
+      task :step1 => :string do |i| i end
+
+      dep :step1, :input1 => 1
+      input :input2, :string
+      task :step2 => :string do |i| step(:step1).load end
+    end
+
+    job = wf.job(:step2, :input1 => 2)
+    assert_equal 1, job.run
+    assert_equal Task::DEFAULT_NAME, job.name
+    assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
+  end
+
+  def test_non_default_inputs
+    wf = Workflow.annonymous_workflow "TaskInputs" do
+      input :input1, :integer, "", 1
+      input :input2, :integer, "", 0
+      task :step1 => :string do |i1,i2| i1 + i2 end
+
+      dep :step1, :input2 => 1
+      task :step2 => :string do |i| step(:step1).load end
+
+      dep :step2
+      task :step3 => :string do |i| step(:step1).load end
+    end
+
+    job = wf.job(:step3, :input1 => 1)
+    assert_equal Task::DEFAULT_NAME, job.name
+    assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
+    assert_equal 2, job.run
+
+    job = wf.job(:step3, :input1 => 2)
+    assert_equal 3, job.run
+
+
+    job = wf.job(:step3)
+    assert_equal Task::DEFAULT_NAME, job.name
+    assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
+  end
+
+  def test_can_fail
+    wf = Workflow.annonymous_workflow "TaskInputs" do
+      input :input1, :integer, "", 1
+      task :step1 => :string do |i1| 
+        if i1 < 0
+          raise ScoutException
+        else
+          i1
+        end
+      end
+
+      dep :step1, :canfail => true
+      task :step2 => :string do |i|
+        if step(:step1).error?
+          0
+        else
+          step(:step1).load 
+        end
+      end
+    end
+
+    assert_equal 1, wf.job(:step2, :input1 => 1).run
+    assert_equal 2, wf.job(:step2, :input1 => 2).run
+    assert_equal 0, wf.job(:step2, :input1 => -2).run
+  end
+
+end
+

data/test/scout/workflow/task/test_inputs.rb

@@ -21,6 +21,11 @@ class TestTaskInput < Test::Unit::TestCase
       task :task => :array do
         inputs
       end
+
+      dep :task
+      task :task2 => :array do
+        inputs
+      end
     end
   end
 
@@ -71,8 +76,8 @@ class TestTaskInput < Test::Unit::TestCase
     task = self.example_task
 
     TmpFile.with_file("2\n3") do |integer_array_file|
-      assert_equal task.digest_inputs(:string => "String", :integer => 2, :integer_array => %w(2 3)),
-        task.digest_inputs(:string => "String", :integer => 2, :integer_array => integer_array_file)
+      assert_equal task.process_inputs(:string => "String", :integer => 2, :integer_array => %w(2 3)).last,
+        task.process_inputs(:string => "String", :integer => 2, :integer_array => integer_array_file).last
     end
   end
 
@@ -83,10 +88,10 @@ class TestTaskInput < Test::Unit::TestCase
 
     TmpFile.with_file("1\n2") do |integer_array_file|
       hash1 = Open.open(integer_array_file) do |f|
-        task.digest_inputs(:string => "String", :integer => 2, :integer_array => f)
+        task.process_inputs(:string => "String", :integer => 2, :integer_array => f).last
       end
       hash2 = Open.open(integer_array_file) do |f|
-        task.digest_inputs(:string => "String", :integer => 2, :integer_array => f)
+        task.process_inputs(:string => "String", :integer => 2, :integer_array => f).last
       end
       assert_equal hash1, hash2
     end
@@ -98,22 +103,23 @@ class TestTaskInput < Test::Unit::TestCase
     task = self.example_task
 
     TmpFile.with_file("2\n3") do |integer_array_file|
-      assert_equal task.digest_inputs(:string => "String", :integer => 2, :integer_array => %w(2 3)),
-        task.digest_inputs(:string => "String", :integer => 2, :integer_array => integer_array_file)
+      assert_equal task.process_inputs(:string => "String", :integer => 2, :integer_array => %w(2 3)).last,
+        task.process_inputs(:string => "String", :integer => 2, :integer_array => integer_array_file).last
     end
   end
 
   def test_save_and_load
     task = self.example_task
 
+    sss 0
     TmpFile.with_file("2\n3") do |integer_array_file|
       inputs = {:string => "String", :integer => 2, :integer_array => integer_array_file, :float_array => %w(1.1 2.2)}
-      original_digest =  task.digest_inputs(inputs)
+      original_digest =  task.process_inputs(inputs).last
 
       TmpFile.with_file do |save_directory|
         task.save_inputs(save_directory, inputs)
         new_inputs = task.load_inputs(save_directory)
-        new_digest =  task.digest_inputs(new_inputs)
+        new_digest =  task.process_inputs(new_inputs).last
         assert_equal original_digest, new_digest
       end
     end
@@ -124,13 +130,13 @@ class TestTaskInput < Test::Unit::TestCase
 
     TmpFile.with_file("TEST") do |somefile|
       inputs = {:string => "String", :integer => 2, :file => somefile, :float_array => %w(1.1 2.2)}
-      original_digest =  task.digest_inputs(inputs)
+      original_digest =  task.process_inputs(inputs).last
 
       TmpFile.with_file do |save_directory|
         task.save_inputs(save_directory, inputs)
         Open.rm somefile
         new_inputs = task.load_inputs(save_directory)
-        new_digest =  task.digest_inputs(new_inputs)
+        new_digest =  task.process_inputs(new_inputs).last
         assert_equal original_digest, new_digest
       end
     end
@@ -141,7 +147,7 @@ class TestTaskInput < Test::Unit::TestCase
 
     TmpFile.with_file("TEST") do |somefile|
       inputs = {:string => "String", :integer => 2, :file => somefile, :float_array => %w(1.1 2.2)}
-      original_digest =  task.digest_inputs(inputs)
+      original_digest =  task.process_inputs(inputs).last
 
       TmpFile.with_file do |save_directory|
         task.save_inputs(save_directory, inputs)
@@ -149,7 +155,7 @@ class TestTaskInput < Test::Unit::TestCase
           Open.cp save_directory, copy_directory
           Open.rm_rf save_directory
           new_inputs = task.load_inputs(copy_directory)
-          new_digest =  task.digest_inputs(new_inputs)
+          new_digest =  task.process_inputs(new_inputs).last
           assert_equal original_digest, new_digest
         end
       end
@@ -167,14 +173,61 @@ class TestTaskInput < Test::Unit::TestCase
       Open.write(file1, "TEST1")
       Open.write(file2, "TEST2")
       inputs = {:string => "String", :integer => 2, :file_array => [file1, file2], :float_array => %w(1.1 2.2)}
-      original_digest =  task.digest_inputs(inputs)
+      original_digest =  task.process_inputs(inputs).last
 
       TmpFile.with_file do |save_directory|
         task.save_inputs(save_directory, inputs)
         Open.rm(file1)
         Open.rm(file2)
         new_inputs = task.load_inputs(save_directory)
-        new_digest =  task.digest_inputs(new_inputs)
+        new_digest =  task.process_inputs(new_inputs).last
+        assert_equal original_digest, new_digest
+      end
+    end
+  end
+
+  def test_save_and_load_file_array_task2
+    task = self.example_workflow.tasks[:task2]
+
+    TmpFile.with_file do |dir|
+      file1 = File.join(dir, 'subdir1/file')
+      file2 = File.join(dir, 'subdir2/file')
+
+      Open.write(file1, "TEST1")
+      Open.write(file2, "TEST2")
+      inputs = {:string => "String", :integer => 2, :file_array => [file1, file2], :float_array => %w(1.1 2.2)}
+      original_digest =  task.process_inputs(inputs).last
+
+      TmpFile.with_file do |save_directory|
+        task.save_inputs(save_directory, inputs)
+        Open.rm(file1)
+        Open.rm(file2)
+        new_inputs = task.load_inputs(save_directory)
+        new_digest =  task.process_inputs(new_inputs).last
+        assert_equal original_digest, new_digest
+      end
+    end
+  end
+
+  def test_save_and_load_array_from_file
+    task = self.example_workflow.tasks[:task]
+
+    TmpFile.with_file do |dir|
+      array = %w(1 2 3 4 5)
+      file1 = File.join(dir, 'subdir1/file')
+      Open.write(file1, array * "\n")
+      inputs = {:array => file1 }
+      original_digest =  task.process_inputs(inputs).last
+
+      TmpFile.with_file do |save_directory|
+        iii inputs
+        task.save_inputs(save_directory, inputs)
+        Open.rm(file1)
+        new_inputs = task.load_inputs(save_directory)
+        iii new_inputs
+        new_digest =  task.process_inputs(new_inputs).last
+        iii new_digest
+        iii original_digest
         assert_equal original_digest, new_digest
       end
     end

data/test/scout/workflow/test_definition.rb

@@ -0,0 +1,18 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestClass < Test::Unit::TestCase
+  def test_function_task
+    wf = Workflow.annonymous_workflow do
+      self.name = "StringLength"
+      def self.str_length(s)
+        s.length
+      end
+
+      input :string, :string
+      task :str_length => :integer
+    end
+    assert_equal 5, wf.job(:str_length, :string => "12345").run
+  end
+end
+

data/test/scout/workflow/test_documentation.rb

@@ -26,5 +26,29 @@ class TestWorkflowDocumentation < Test::Unit::TestCase
     assert_match 'test', UsageWorkflow.documentation[:title]
     assert_match 'presented', UsageWorkflow.documentation[:description]
   end
+
+  def test_documentation_markdown
+    doc =<<-EOF
+summary
+
+description
+
+# Tasks
+
+## task1
+
+task 1 summary
+
+task 1 description
+
+## task2
+task 2 summary
+
+task 2 description
+    EOF
+
+    assert_includes Workflow.parse_workflow_doc(doc)[:tasks], 'task1'
+    assert_includes Workflow.parse_workflow_doc(doc)[:tasks]['task2'], 'task 2 description'
+  end
 end
 

data/test/scout/workflow/test_step.rb

@@ -47,6 +47,17 @@ class TestWorkflowStep < Test::Unit::TestCase
     end
     step1.type = :array
 
+    step1.clean
+    res =  step1.run(false)
+    refute IO === res
+    step1.join
+
+    step1.clean
+    res =  step1.run(true)
+    assert IO === res
+    step1.join
+    step1.clean
+
     step2 = Step.new tmpfile.step2 do 
       step1 = dependencies.first
       stream = step1.stream
@@ -75,7 +86,7 @@ class TestWorkflowStep < Test::Unit::TestCase
     end
     assert_equal times/2, lines.length
 
-    stream = step2.run
+    stream = step2.run(true)
     assert step1.streaming?
     assert step2.streaming?
 
@@ -137,7 +148,7 @@ class TestWorkflowStep < Test::Unit::TestCase
 
     step3.recursive_clean
 
-    stream = step3.run
+    stream = step3.run(true)
     out = []
     while l = stream.gets
       out << l
@@ -218,7 +229,7 @@ class TestWorkflowStep < Test::Unit::TestCase
     step4.dependencies = [step2, step3]
 
     lines = []
-    io = step4.run
+    io = step4.run(true)
     Log::ProgressBar.with_bar severity: 0 do |b|
       while line = io.gets
         b.tick
@@ -229,4 +240,102 @@ class TestWorkflowStep < Test::Unit::TestCase
 
     assert_equal times, lines.length
   end
+
+  def test_fork_stream_fork
+    tmpfile = tmpdir.test_step
+
+    times = 10_000
+    sleep = 0.1 / times
+
+    step1 = Step.new tmpfile.step1, [times, sleep] do |times,sleep|
+      sleep 1
+      Open.open_pipe do |sin|
+        times.times do |i|
+          sin.puts "line-#{i}"
+          sleep sleep
+        end
+      end
+    end
+    step1.type = :array
+
+
+    step2 = Step.new tmpfile.step2 do 
+      step1 = dependencies.first
+      stream = step1.stream
+
+      Open.open_pipe do |sin|
+        while line = stream.gets
+          num = line.split("-").last
+          next if num.to_i % 2 == 1
+          sin.puts "S2: " + line
+        end
+      end
+    end
+    step2.type = :array
+    step2.dependencies = [step1]
+
+    step3 = Step.new tmpfile.step3 do 
+      step1 = dependencies.first
+      stream = step1.stream
+
+      Open.open_pipe do |sin|
+        while line = stream.gets
+          num = line.split("-").last
+          next if num.to_i % 2 == 0
+          sin.puts "S3: " + line
+        end
+      end
+    end
+    step3.type = :array
+    step3.dependencies = [step1]
+
+
+    step4 = Step.new tmpfile.step4 do 
+      step2, step3 = dependencies
+
+      mutex = Mutex.new
+      Open.open_pipe do |sin|
+        t2 = Thread.new do
+          stream2 = step2.stream
+          while line = stream2.gets
+            sin.puts line
+          end
+        end
+
+        t3 = Thread.new do
+          stream3 = step3.stream
+          while line = stream3.gets
+            sin.puts line
+          end
+        end
+        t2.join
+        t3.join
+      end
+    end
+    step4.type = :array
+    step4.dependencies = [step2, step3]
+
+    step4.recursive_clean
+    step4.fork
+    assert Array === step4.load
+    assert_equal times, step4.load.size
+  end
+
+  def test_dependency_canfail
+    tmpfile = tmpdir.test_step
+    step1 = Step.new tmpfile.step1, ["12"] do |s|
+      s.length
+    end
+
+    step2 = Step.new tmpfile.step2 do 
+      step1 = dependencies.first
+      step1.inputs.first + " has " + step1.load.to_s + " characters"
+    end
+
+    step2.dependencies = [step1]
+
+    assert_equal "12 has 2 characters", step2.run
+    assert_equal "12 has 2 characters", step2.run
+  end
+
 end