scout-gear 8.0.0 → 8.1.0

data/test/scout/workflow/step/test_dependencies.rb

@@ -21,5 +21,73 @@ class TestStepDependencies < Test::Unit::TestCase
     assert_equal 2, step2.inputs[:input2]
     assert_equal "12", step2.recursive_inputs[:input1]
   end
+
+  def test_can_fail
+    tmpfile = tmpdir.test_step
+    step1 = Step.new tmpfile.step1, NamedArray.setup(["12"], %w(input1)) do |s|
+      raise ScoutException
+    end
+
+    step2 = Step.new tmpfile.step2, NamedArray.setup([2], %w(input2)) do |times|
+      step1 = dependencies.first
+      if step1.error?
+        (step1.inputs.first + " has unknown characters") * times
+      else
+        (step1.inputs.first + " has " + step1.load.to_s + " characters") * times
+      end
+    end
+
+    step2.dependencies = [step1]
+    step2.compute = {step1.path => [:canfail] }
+
+    assert_include step2.run, "unknown"
+  end
+
+  def test_can_stream
+    tmpfile = tmpdir.test_step
+
+    Misc.with_env "SCOUT_EXPLICIT_STREAMING", "true" do
+      times = 10_000
+      sleep = 1 / times
+
+      step1 = Step.new tmpfile.step1, [times, sleep] do |times,sleep|
+        Open.open_pipe do |sin|
+          times.times do |i|
+            sin.puts "line-#{i}"
+            sleep sleep
+          end
+        end
+      end
+      step1.type = :array
+
+      step2 = Step.new tmpfile.step2 do 
+        step1 = dependencies.first
+        raise ScoutException unless step1.streaming?
+        stream = step1.stream
+
+        Open.open_pipe do |sin|
+          while line = stream.gets
+            num = line.split("-").last
+            next if num.to_i % 2 == 1
+            sin.puts line
+          end
+        end
+      end
+      step2.type = :array
+      step2.dependencies = [step1]
+
+      assert_raise ScoutException do
+        step2.run
+      end
+
+      step2.recursive_clean
+
+      step2.compute = {step1.path => [:stream]}
+
+      assert_nothing_raised do
+        step2.run
+      end
+    end
+  end
 end
 

data/test/scout/workflow/step/test_info.rb

@@ -25,4 +25,22 @@ class TestStepInfo < Test::Unit::TestCase
       assert_equal "12 has 2 characters", step2.run
     end
   end
+
+  def test_inputs_marshal
+    TmpFile.with_file do |tmpdir|
+      Path.setup(tmpdir)
+      tmpfile = tmpdir.test_step
+
+      path = tmpfile.foo
+      step1 = Step.new tmpfile.step1, [path] do |s|
+        s.length
+      end
+
+      step1.run
+
+      refute Path === step1.info[:inputs][0]
+
+    end
+
+  end
 end

data/test/scout/workflow/step/test_status.rb

@@ -15,7 +15,6 @@ class TestStepStatus < Test::Unit::TestCase
 
     step2.dependencies = [step1]
 
-    sss 0
     Misc.with_env "SCOUT_UPDATE", "true" do
       step2.run
       assert step2.updated?

data/test/scout/workflow/task/test_dependencies.rb

@@ -0,0 +1,355 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/workflow'
+class TestTaskDependencies < Test::Unit::TestCase
+  def test_task_override_dep_exec
+    wf = Workflow.annonymous_workflow "TaskInputs" do
+      input :input1, :integer
+      task :step1 => :integer do |i| i end
+
+      dep :step1
+      input :input2, :integer, "Integer", 3
+      task :step2 => :integer do |i| i * step(:step1).load end
+    end
+
+    assert_equal 6, wf.job(:step2, :input1 => 2, :input2 => 3).exec
+
+    step1_job = wf.job(:step1, :input1 => 6)
+    assert_equal 18, wf.job(:step2, :input1 => 2, "TaskInputs#step1" => step1_job).exec
+
+    assert_equal 18, wf.job(:step2, :input1 => 2, "TaskInputs#step1" => step1_job).exec
+
+    assert_equal 18, wf.job(:step2, "TaskInputs#step1" => step1_job).exec
+  end
+
+  def test_task_override_dep
+    wf = Workflow.annonymous_workflow "TaskInputs" do
+      input :input1, :integer
+      task :step1 => :integer do |i| i end
+
+      dep :step1
+      input :input2, :integer, "Integer", 3
+      task :step2 => :integer do |i| i * step(:step1).load end
+    end
+
+    assert_equal 6, wf.job(:step2, :input1 => 2, :input2 => 3).run
+
+    step1_job = wf.job(:step1, :input1 => 6)
+    assert_equal 18, wf.job(:step2, :input1 => 2, "TaskInputs#step1" => step1_job).run
+
+    assert_equal 18, wf.job(:step2, :input1 => 2, "TaskInputs#step1" => step1_job).run
+
+    assert_equal 18, wf.job(:step2, "TaskInputs#step1" => step1_job).run
+  end
+
+  def test_input_dep
+    wf = Workflow.annonymous_workflow "TaskInputs" do
+      input :v1, :integer
+      input :v2, :integer
+      task :sum => :integer do |v1,v2|
+        v1 + v2
+      end
+
+      input :input1, :integer
+      task :step1 => :integer do |i| i end
+
+      input :input2, :integer
+      task :step2 => :integer do |i| i end
+
+      dep :step1, :input1 => 2
+      dep :step2, :input2 => 3
+      dep :sum, :v1 => :step1, :v2 => :step2
+      task :my_sum => :integer do
+        dependencies.last.load
+      end
+
+      dep :my_sum
+      task :double => :integer do
+        step(:my_sum).load * 2 
+      end
+    end
+
+    job = wf.job(:my_sum)
+    assert_equal 5, job.run
+    assert_equal Task::DEFAULT_NAME, job.name
+
+    job = wf.job(:double)
+    assert_equal Task::DEFAULT_NAME, job.name
+
+    step1 = wf.job(:step1, :input1 => 3)
+    assert_equal 3, step1.run
+    job = wf.job(:double, "TaskInputs#step1" => step1)
+    assert_equal 12, job.run
+    assert_not_equal Task::DEFAULT_NAME, job.name
+
+    step1 = wf.job(:step1, :input1 => 4)
+    assert_equal 4, step1.run
+    job = wf.job(:double, "TaskInputs#step1" => step1)
+    assert_equal 14, job.run
+    assert_not_equal Task::DEFAULT_NAME, job.name
+  end
+
+  def test_input_dep_override
+    wf = Workflow.annonymous_workflow "TaskInputs" do
+      input :v1, :integer
+      input :v2, :integer
+      task :sum => :integer do |v1,v2|
+        v1 + v2
+      end
+
+      input :input1, :integer
+      task :step1 => :integer do |i| i end
+
+      input :input2, :integer
+      task :step2 => :integer do |i| i end
+
+      dep :step1
+      dep :step2
+      task :my_sum => :integer do
+        dependencies.inject(0){|acc,d| acc += d.load }
+      end
+    end
+
+    step2 = wf.job(:step2, :input2 => 4)
+    job = wf.job(:my_sum, :input1 => 2, :input2 => 3, "TaskInputs#step2"=> step2)
+    assert_equal 6, job.exec
+
+    job = wf.job(:my_sum, :input1 => 2, :input2 => 3)
+    assert_equal 5, job.run
+
+    TmpFile.with_file(4) do |file|
+      job = wf.job(:my_sum, :input1 => 2, :input2 => 3, "TaskInputs#step2"=> file)
+      assert_equal 6, job.run
+      assert_not_equal Task::DEFAULT_NAME, job.name
+    end
+  end
+
+  def test_input_rename
+    wf = Workflow.annonymous_workflow "TaskInputs" do
+      input :v1, :integer
+      input :v2, :integer
+      task :sum => :integer do |v1,v2|
+        v1 + v2
+      end
+
+      input :vv1, :integer
+      input :vv2, :integer
+      dep :sum, :v1 => :vv1, :v2 => :vv2
+      task :my_sum => :integer do
+        dependencies.last.load
+      end
+    end
+
+    job = wf.job(:my_sum, :vv1 => 2, :vv2 => 3)
+    assert_equal 5, job.run
+  end
+
+  def test_defaults_in_dep_block
+    wf = Workflow.annonymous_workflow "TaskInputs" do
+      input :v1, :integer
+      input :v2, :integer
+      task :sum => :integer do |v1,v2|
+        v1 + v2
+      end
+
+      input :vv1, :integer
+      input :vv2, :integer, nil, 3
+      dep :sum, :v1 => :placeholder, :v2 => :placeholder do |jobname,options,dependencies|
+        raise "Non-numeric value where integer expected" unless Numeric === options[:vv1]
+        {inputs: {v1: options[:vv1], v2: options[:vv2]} }
+      end
+      task :my_sum => :integer do
+        dependencies.last.load
+      end
+    end
+
+    job = wf.job(:my_sum, :vv1 => "2")
+    assert_equal 5, job.run
+  end
+
+  def test_dependency_jobname
+    wf = Workflow.annonymous_workflow "TaskInputs" do
+      input :v1, :integer
+      input :v2, :integer
+      task :sum => :integer do |v1,v2|
+        v1 + v2
+      end
+
+      input :vv1, :integer
+      input :vv2, :integer
+      dep :sum, :v1 => :vv1, :v2 => :vv2, :jobname => "OTHER_NAME"
+      task :my_sum => :integer do
+        dependencies.last.load
+      end
+    end
+
+    job = wf.job(:my_sum, "TEST_NAME", :vv1 => 2, :vv2 => 3)
+    assert_equal 5, job.run
+    assert_equal "TEST_NAME", job.clean_name
+    assert_equal "OTHER_NAME", job.step(:sum).clean_name
+  end
+
+  def test_no_param_last_job
+    wf = Workflow.annonymous_workflow "TaskInputs" do
+      input :v1, :integer
+      input :v2, :integer
+      task :sum => :integer do |v1,v2|
+        v1 + v2
+      end
+
+      dep :sum
+      task :my_sum => :integer do
+        dependencies.last.load
+      end
+    end
+
+    job = wf.job(:my_sum, :v1 => 2, :v2 => "3")
+    assert_equal 5, job.run
+    refute_equal Task::DEFAULT_NAME, job.name
+  end
+
+  def test_no_param_last_job_block
+    wf = Workflow.annonymous_workflow "TaskInputs" do
+      input :v1, :integer
+      input :v2, :integer
+      task :sum => :integer do |v1,v2|
+        v1 + v2
+      end
+
+      dep :sum do |jobname,options|
+        {inputs: options}
+      end
+      task :my_sum => :integer do
+        dependencies.last.load
+      end
+    end
+
+    job = wf.job(:my_sum, :v1 => 2, :v2 => "3")
+    assert_equal 5, job.run
+    refute_equal Task::DEFAULT_NAME, job.name
+  end
+
+
+  def test_override_inputs_block
+    wf = Workflow.annonymous_workflow "TaskInputs" do
+      input :input1, :string
+      task :step1 => :string do |i| i end
+
+      dep :step1, :input1 => 1  do |id,options|
+        {:inputs => options}
+      end
+      task :step2 => :string do |i| step(:step1).load end
+    end
+
+    job = wf.job(:step2, :input1 => 2)
+    assert_equal 1, job.run
+    assert_equal Task::DEFAULT_NAME, job.name
+    assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
+  end
+
+  def test_default_inputs_in_block
+    wf = Workflow.annonymous_workflow "TaskInputs" do
+      input :input1, :string
+      task :step1 => :string do |i| i end
+
+      dep :step1  do |id,options|
+        {}
+      end
+      task :step2 => :string do |i| step(:step1).load end
+    end
+
+    job = wf.job(:step2, "SOME_NAME", :input1 => 2)
+    assert_equal "SOME_NAME", job.step(:step1).clean_name
+    assert_equal 2, job.run
+  end
+
+  def test_override_inputs_block_array
+    wf = Workflow.annonymous_workflow "TaskInputs" do
+      input :input1, :string
+      task :step1 => :string do |i| i end
+
+      dep :step1, :input1 => 1  do |id,options|
+        [{:inputs => options}]
+      end
+      input :input2, :string
+      task :step2 => :string do |i| step(:step1).load end
+    end
+
+    job = wf.job(:step2, :input1 => 2)
+    assert_equal 1, job.run
+    assert_equal Task::DEFAULT_NAME, job.name
+    assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
+  end
+
+  def test_override_inputs
+    wf = Workflow.annonymous_workflow "TaskInputs" do
+      input :input1, :string
+      task :step1 => :string do |i| i end
+
+      dep :step1, :input1 => 1
+      input :input2, :string
+      task :step2 => :string do |i| step(:step1).load end
+    end
+
+    job = wf.job(:step2, :input1 => 2)
+    assert_equal 1, job.run
+    assert_equal Task::DEFAULT_NAME, job.name
+    assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
+  end
+
+  def test_non_default_inputs
+    wf = Workflow.annonymous_workflow "TaskInputs" do
+      input :input1, :integer, "", 1
+      input :input2, :integer, "", 0
+      task :step1 => :string do |i1,i2| i1 + i2 end
+
+      dep :step1, :input2 => 1
+      task :step2 => :string do |i| step(:step1).load end
+
+      dep :step2
+      task :step3 => :string do |i| step(:step1).load end
+    end
+
+    job = wf.job(:step3, :input1 => 1)
+    assert_equal Task::DEFAULT_NAME, job.name
+    assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
+    assert_equal 2, job.run
+
+    job = wf.job(:step3, :input1 => 2)
+    assert_equal 3, job.run
+
+
+    job = wf.job(:step3)
+    assert_equal Task::DEFAULT_NAME, job.name
+    assert_not_equal Task::DEFAULT_NAME, job.step(:step1).name
+  end
+
+  def test_can_fail
+    wf = Workflow.annonymous_workflow "TaskInputs" do
+      input :input1, :integer, "", 1
+      task :step1 => :string do |i1| 
+        if i1 < 0
+          raise ScoutException
+        else
+          i1
+        end
+      end
+
+      dep :step1, :canfail => true
+      task :step2 => :string do |i|
+        if step(:step1).error?
+          0
+        else
+          step(:step1).load 
+        end
+      end
+    end
+
+    assert_equal 1, wf.job(:step2, :input1 => 1).run
+    assert_equal 2, wf.job(:step2, :input1 => 2).run
+    assert_equal 0, wf.job(:step2, :input1 => -2).run
+  end
+
+end
+

data/test/scout/workflow/task/test_inputs.rb

@@ -21,6 +21,11 @@ class TestTaskInput < Test::Unit::TestCase
       task :task => :array do
         inputs
       end
+
+      dep :task
+      task :task2 => :array do
+        inputs
+      end
     end
   end
 
@@ -106,6 +111,7 @@ class TestTaskInput < Test::Unit::TestCase
   def test_save_and_load
     task = self.example_task
 
+    sss 0
     TmpFile.with_file("2\n3") do |integer_array_file|
       inputs = {:string => "String", :integer => 2, :integer_array => integer_array_file, :float_array => %w(1.1 2.2)}
       original_digest =  task.process_inputs(inputs).last
@@ -179,4 +185,51 @@ class TestTaskInput < Test::Unit::TestCase
       end
     end
   end
+
+  def test_save_and_load_file_array_task2
+    task = self.example_workflow.tasks[:task2]
+
+    TmpFile.with_file do |dir|
+      file1 = File.join(dir, 'subdir1/file')
+      file2 = File.join(dir, 'subdir2/file')
+
+      Open.write(file1, "TEST1")
+      Open.write(file2, "TEST2")
+      inputs = {:string => "String", :integer => 2, :file_array => [file1, file2], :float_array => %w(1.1 2.2)}
+      original_digest =  task.process_inputs(inputs).last
+
+      TmpFile.with_file do |save_directory|
+        task.save_inputs(save_directory, inputs)
+        Open.rm(file1)
+        Open.rm(file2)
+        new_inputs = task.load_inputs(save_directory)
+        new_digest =  task.process_inputs(new_inputs).last
+        assert_equal original_digest, new_digest
+      end
+    end
+  end
+
+  def test_save_and_load_array_from_file
+    task = self.example_workflow.tasks[:task]
+
+    TmpFile.with_file do |dir|
+      array = %w(1 2 3 4 5)
+      file1 = File.join(dir, 'subdir1/file')
+      Open.write(file1, array * "\n")
+      inputs = {:array => file1 }
+      original_digest =  task.process_inputs(inputs).last
+
+      TmpFile.with_file do |save_directory|
+        iii inputs
+        task.save_inputs(save_directory, inputs)
+        Open.rm(file1)
+        new_inputs = task.load_inputs(save_directory)
+        iii new_inputs
+        new_digest =  task.process_inputs(new_inputs).last
+        iii new_digest
+        iii original_digest
+        assert_equal original_digest, new_digest
+      end
+    end
+  end
 end

data/test/scout/workflow/test_definition.rb

@@ -0,0 +1,18 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestClass < Test::Unit::TestCase
+  def test_function_task
+    wf = Workflow.annonymous_workflow do
+      self.name = "StringLength"
+      def self.str_length(s)
+        s.length
+      end
+
+      input :string, :string
+      task :str_length => :integer
+    end
+    assert_equal 5, wf.job(:str_length, :string => "12345").run
+  end
+end
+

data/test/scout/workflow/test_documentation.rb

@@ -26,5 +26,29 @@ class TestWorkflowDocumentation < Test::Unit::TestCase
     assert_match 'test', UsageWorkflow.documentation[:title]
     assert_match 'presented', UsageWorkflow.documentation[:description]
   end
+
+  def test_documentation_markdown
+    doc =<<-EOF
+summary
+
+description
+
+# Tasks
+
+## task1
+
+task 1 summary
+
+task 1 description
+
+## task2
+task 2 summary
+
+task 2 description
+    EOF
+
+    assert_includes Workflow.parse_workflow_doc(doc)[:tasks], 'task1'
+    assert_includes Workflow.parse_workflow_doc(doc)[:tasks]['task2'], 'task 2 description'
+  end
 end
 

data/test/scout/workflow/test_step.rb

@@ -47,6 +47,17 @@ class TestWorkflowStep < Test::Unit::TestCase
     end
     step1.type = :array
 
+    step1.clean
+    res =  step1.run(false)
+    refute IO === res
+    step1.join
+
+    step1.clean
+    res =  step1.run(true)
+    assert IO === res
+    step1.join
+    step1.clean
+
     step2 = Step.new tmpfile.step2 do 
       step1 = dependencies.first
       stream = step1.stream
@@ -229,4 +240,102 @@ class TestWorkflowStep < Test::Unit::TestCase
 
     assert_equal times, lines.length
   end
+
+  def test_fork_stream_fork
+    tmpfile = tmpdir.test_step
+
+    times = 10_000
+    sleep = 0.1 / times
+
+    step1 = Step.new tmpfile.step1, [times, sleep] do |times,sleep|
+      sleep 1
+      Open.open_pipe do |sin|
+        times.times do |i|
+          sin.puts "line-#{i}"
+          sleep sleep
+        end
+      end
+    end
+    step1.type = :array
+
+
+    step2 = Step.new tmpfile.step2 do 
+      step1 = dependencies.first
+      stream = step1.stream
+
+      Open.open_pipe do |sin|
+        while line = stream.gets
+          num = line.split("-").last
+          next if num.to_i % 2 == 1
+          sin.puts "S2: " + line
+        end
+      end
+    end
+    step2.type = :array
+    step2.dependencies = [step1]
+
+    step3 = Step.new tmpfile.step3 do 
+      step1 = dependencies.first
+      stream = step1.stream
+
+      Open.open_pipe do |sin|
+        while line = stream.gets
+          num = line.split("-").last
+          next if num.to_i % 2 == 0
+          sin.puts "S3: " + line
+        end
+      end
+    end
+    step3.type = :array
+    step3.dependencies = [step1]
+
+
+    step4 = Step.new tmpfile.step4 do 
+      step2, step3 = dependencies
+
+      mutex = Mutex.new
+      Open.open_pipe do |sin|
+        t2 = Thread.new do
+          stream2 = step2.stream
+          while line = stream2.gets
+            sin.puts line
+          end
+        end
+
+        t3 = Thread.new do
+          stream3 = step3.stream
+          while line = stream3.gets
+            sin.puts line
+          end
+        end
+        t2.join
+        t3.join
+      end
+    end
+    step4.type = :array
+    step4.dependencies = [step2, step3]
+
+    step4.recursive_clean
+    step4.fork
+    assert Array === step4.load
+    assert_equal times, step4.load.size
+  end
+
+  def test_dependency_canfail
+    tmpfile = tmpdir.test_step
+    step1 = Step.new tmpfile.step1, ["12"] do |s|
+      s.length
+    end
+
+    step2 = Step.new tmpfile.step2 do 
+      step1 = dependencies.first
+      step1.inputs.first + " has " + step1.load.to_s + " characters"
+    end
+
+    step2.dependencies = [step1]
+
+    assert_equal "12 has 2 characters", step2.run
+    assert_equal "12 has 2 characters", step2.run
+  end
+
 end