scout-gear 1.2.0 → 5.1.1

data/test/scout/test_open.rb

@@ -0,0 +1,146 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestOpen < Test::Unit::TestCase
+
+  def test_read
+    content =<<-EOF
+1
+2
+3
+4
+    EOF
+    TmpFile.with_file(content) do |file|
+      sum = 0
+      Open.read file do |line| sum += line.to_i end
+      assert_equal(1 + 2 + 3 + 4, sum)
+      assert_equal(content, Open.read(file))
+    end
+  end
+
+  def test_gzip
+    content =<<-EOF
+1
+2
+3
+4
+    EOF
+    TmpFile.with_file(content) do |file|
+      `gzip #{file}`
+      assert_equal(content, Open.read(file + '.gz'))
+      FileUtils.rm file + '.gz'
+    end
+  end
+
+
+
+  #def test_wget
+  #  assert(Misc.fixutf8(Open.wget('http://google.com', :quiet => true).read) =~ /html/)
+  #end
+
+  #def test_nice
+  #  nice =  0.5
+
+  #  t = Time.now
+  #  Open.wget('http://google.com', :quiet => true, :nice => nice).read
+  #  assert(Time.now - t + 0.5 >= nice)
+
+  #  Open.wget('http://google.com', :quiet => true, :nice => nice, :nice_key => 1).read
+  #  t = Time.now
+  #  Open.wget('http://google.com', :quiet => true, :nice => nice, :nice_key => 1).read
+  #  assert(Time.now - t + 0.5 >= nice)
+  #end
+
+  #def test_remote?
+  #  assert(Open.remote?('http://google.com'))
+  #  assert(! Open.remote?('~/.bashrc'))
+  #end
+
+  #def test_open
+  #  assert(Open.read('http://google.com', :quiet => true, :nocache => :update) =~ /html/)
+  #end
+
+  #def test_repo_dir
+  #  file1 = "TEST"
+  #  file2 = "TEST" * 1000
+  #  TmpFile.with_file do |tmpdir|
+  #    tmpdir = Rbbt.tmp.repo_dir.find
+  #    normal = File.join(tmpdir, 'normal')
+  #    repo = File.join(tmpdir, 'repo')
+
+  #    Open.repository_dirs.push(repo)
+
+  #    Misc.benchmark(100) do
+  #      filename = "file" << (rand * 100).to_i.to_s
+  #      Open.write(File.join(normal, filename), file2)
+  #      100.times do 
+  #        Open.read(File.join(normal,  filename))
+  #      end
+  #    end
+
+  #    Misc.benchmark(100) do
+  #      filename = "file" << (rand * 100).to_i.to_s
+  #      Open.write(File.join(repo, filename), file2)
+  #      100.times do 
+  #        Open.read(File.join(repo, filename))
+  #      end
+  #    end
+  #  end
+  #end
+  #
+  #def test_repo_dir2
+  #  TmpFile.with_file do |tmpdir|
+  #    tmpdir = Rbbt.tmp.repo_dir.find
+  #    repo = File.join(tmpdir, 'repo')
+
+  #    Open.repository_dirs.push(repo)
+
+  #    obj = { :a => "???a"}
+  #    filename = "file" << (rand * 100).to_i.to_s
+  #    Open.write(File.join(repo, filename), Marshal.dump(obj))
+  #    dump = Open.read(File.join(repo, filename))
+  #    obj_cp = Marshal.load(dump)
+  #    assert_equal obj, obj_cp
+  #  end
+  #end
+
+  #def test_repo_marshal
+  #  TmpFile.with_file do |tmpdir|
+  #    tmpdir = Rbbt.tmp.repo_dir.find
+  #    repo = File.join(tmpdir, 'repo')
+
+  #    filename = 'file'
+  #    Open.repository_dirs.push(repo)
+
+  #    obj = {:a => "string", :pid => nil, :num => 1000, :p  => Rbbt.tmp.foo}
+  #    Open.write(File.join(repo, filename), Marshal.dump(obj))
+  #    new =Open.open(File.join(repo, filename)) do |f|
+  #      Marshal.load(f)
+  #    end
+
+  #    assert_equal new, obj
+  #  end
+
+  #end
+
+  #def test_write_stream_repo
+  #  TmpFile.with_file do |tmpdir|
+  #    tmpdir = Rbbt.tmp.repo_dir.find
+  #    repo = File.join(tmpdir, 'repo')
+
+  #    file = File.join(repo, 'file')
+  #    Open.repository_dirs.push(repo)
+
+  #    text = (["text"] * 5) * "\n"
+
+  #    Misc.consume_stream(StringIO.new(text), false, file)
+
+  #    assert_equal text, Open.read(file)
+  #    assert !File.exist?(file)
+  #    assert Open.exists? file
+  #  end
+
+  #end
+
+end
+

data/test/scout/test_path.rb

@@ -37,8 +37,10 @@ class TestPath < Test::Unit::TestCase
     TmpFile.with_file do |tmpdir|
       Path.setup tmpdir
       FileUtils.mkdir_p tmpdir.lib
+      lib_path = File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+      lib_path = File.join(Path.caller_lib_dir, 'lib', lib_path)
       File.write tmpdir.lib.file, <<-EOR 
-require '#{File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')}'
+require '#{lib_path}'
 a = "1"
 Path.setup(a)
 print a.libdir

data/test/scout/test_persist.rb

@@ -0,0 +1,83 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestPersist < Test::Unit::TestCase
+  def test_string
+    TmpFile.with_file do |tmpfile|
+      Path.setup(tmpfile)
+      obj = "TEST"
+      type = :string
+      refute tmpdir.persist.glob("*").any?
+      assert_equal obj, Persist.persist(tmpfile, type, :dir => tmpdir.persist){ obj }
+      assert tmpdir.persist.glob("*").any?
+      assert_equal obj, Persist.persist(tmpfile, type, :dir => tmpdir.persist){ raise "Error" }
+    end
+  end
+
+  def test_float_array
+    TmpFile.with_file do |tmpfile|
+      Path.setup(tmpfile)
+      obj = [1.2,2.2]
+      type = :float_array
+      refute tmpdir.persist.glob("*").any?
+      assert_equal obj, Persist.persist(tmpfile, type, :dir => tmpdir.persist){ obj }
+      assert tmpdir.persist.glob("*").any?
+      assert_equal obj, Persist.persist(tmpfile, type, :dir => tmpdir.persist){ raise "Error" }
+    end
+  end
+
+  def test_string_update
+    TmpFile.with_file do |tmpfile|
+      Path.setup(tmpfile)
+      obj = "TEST"
+      type = :string
+      refute tmpdir.persist.glob("*").any?
+      assert_equal obj, Persist.persist(tmpfile, type, :dir => tmpdir.persist){ obj }
+      assert tmpdir.persist.glob("*").any?
+      assert_raises ScoutException do 
+        Persist.persist(tmpfile, type, :dir => tmpdir.persist, :update => true){ raise ScoutException }
+      end
+      assert_raises ScoutException do 
+        Persist.persist(tmpfile, type, :persist_dir => tmpdir.persist, :persist_update => true){ raise ScoutException }
+      end
+    end
+  end
+
+  def test_stream
+    TmpFile.with_file do |tmpfile|
+      Path.setup(tmpfile)
+      obj = "TEST\nTEST"
+      stream = StringIO.new obj
+      stream.rewind
+      res = Persist.persist(tmpfile, :string, :dir => tmpdir.persist){ stream }
+      assert IO === res
+      assert_equal obj, res.read
+      assert_equal obj, Persist.persist(tmpfile, :string, :dir => tmpdir.persist){ raise ScoutException }
+    end
+  end
+
+  def test_update_time
+    TmpFile.with_file do |dir|
+      Path.setup(dir)
+      obj = "TEST"
+      type = :string
+
+      Open.write(dir.file, "TEST")
+      assert_equal "TEST", Persist.persist(dir.cache, type, :dir => tmpdir.persist){ Open.read(dir.file) }
+      Open.rm(dir.file)
+      assert_equal "TEST", Persist.persist(dir.cache, type, :dir => tmpdir.persist){ Open.read(dir.file) }
+
+      sleep 1
+      Open.write(dir.file2, "TEST2")
+      assert_equal "TEST", Persist.persist(dir.cache, type, :dir => tmpdir.persist){ Open.read(dir.file2) }
+      assert_equal "TEST2", Persist.persist(dir.cache, type, :dir => tmpdir.persist, :update => dir.file2){ Open.read(dir.file2) }
+
+      sleep 1
+      Open.write(dir.file3, "TEST3")
+      sleep 1
+      Open.touch tmpdir.persist.glob("*").first
+      assert_equal "TEST2", Persist.persist(dir.cache, type, :dir => tmpdir.persist, :update => dir.file3){ Open.read(dir.file3) }
+    end
+  end
+end
+

data/test/scout/test_resource.rb

@@ -0,0 +1,26 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestResourceUnit < Test::Unit::TestCase
+  module TestResource
+    extend Resource
+
+    self.subdir = Path.setup('tmp/test-resource')
+  end
+
+
+  def test_root
+
+    p = TestResource.root.some_file
+    assert p.find(:user).include?(ENV["HOME"])
+  end
+
+  def __test_identify
+    assert_equal 'etc/', Rbbt.identify(File.join(ENV["HOME"], '.rbbt/etc/'))
+    assert_equal 'share/databases/', Rbbt.identify('/usr/local/share/rbbt/databases/')
+    assert_equal 'share/databases/DATABASE', Rbbt.identify('/usr/local/share/rbbt/databases/DATABASE')
+    assert_equal 'share/databases/DATABASE/FILE', Rbbt.identify('/usr/local/share/rbbt/databases/DATABASE/FILE')
+    assert_equal 'share/databases/DATABASE/FILE', Rbbt.identify(File.join(ENV["HOME"], '.rbbt/share/databases/DATABASE/FILE'))
+    assert_equal 'share/databases/DATABASE/FILE', Rbbt.identify('/usr/local/share/rbbt/databases/DATABASE/FILE')
+  end
+end

data/test/scout/test_workflow.rb

@@ -0,0 +1,87 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestWorkflow < Test::Unit::TestCase
+
+  module Pantry
+    extend Resource
+    self.subdir = 'share/pantry'
+
+    Pantry.claim Pantry.eggs, :proc do
+      Log.info "Buying Eggs in the store"
+      "Eggs"
+    end
+
+    Pantry.claim Pantry.flour, :proc do
+      Log.info "Buying Flour in the store"
+      "Flour"
+    end
+
+    Pantry.claim Pantry.blueberries, :proc do
+      Log.info "Buying Bluberries in the store"
+      "Bluberries"
+    end
+  end
+
+  module Baking
+    def self.whisk(eggs)
+      "Whisking eggs from #{eggs}"
+    end
+
+    def self.mix(base, mixer)
+      "Mixing base (#{base}) with mixer (#{mixer})"
+    end
+
+    def self.bake(batter)
+      "Baking batter (#{batter})"
+    end
+  end
+
+  module Baking
+    extend Workflow
+
+    dep :prepare_batter
+    task :bake_muffin_tray => :string do 
+      Baking.bake(step(:prepare_batter).load)
+    end
+
+    dep :whisk_eggs
+    input :add_bluberries, :boolean
+    task :prepare_batter => :string do |add_bluberries|
+      whisked_eggs = step(:whisk_eggs).load
+      batter = Baking.mix(whisked_eggs, Pantry.flour.produce)
+
+      if add_bluberries
+        batter = Baking.mix(batter, Pantry.blueberries.produce) 
+      end
+
+      batter
+    end
+
+    task :whisk_eggs => :string do
+      Baking.whisk(Pantry.eggs.produce)
+    end
+  end
+
+  setup do
+    Baking.directory = tmpdir.var.jobs.baking.find
+  end
+
+  def test_task
+    wf = Workflow.annonymous_workflow do
+      task :length => :integer do
+        self.length
+      end
+    end
+    bindings = "12345"
+    assert_equal 5, wf.tasks[:length].exec_on(bindings)
+  end
+
+  def test_baking
+    assert_equal "Baking batter (Mixing base (Mixing base (Whisking eggs from share/pantry/eggs) with mixer (share/pantry/flour)) with mixer (share/pantry/blueberries))",
+      Baking.job(:bake_muffin_tray, :add_bluberries => true).run
+
+    assert_equal "Baking batter (Mixing base (Whisking eggs from share/pantry/eggs) with mixer (share/pantry/flour))",
+      Baking.job(:bake_muffin_tray).run
+  end
+end

data/test/scout/workflow/step/test_info.rb

@@ -0,0 +1,28 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+require 'scout/workflow'
+
+class TestStepInfo < Test::Unit::TestCase
+  def test_dependency
+    Log.with_severity 0 do
+      tmpfile = tmpdir.test_step
+      step1 = Step.new tmpfile.step1, ["12"] do |s|
+        s.length
+      end
+
+      assert_equal 2, step1.exec
+      assert_equal 2, step1.run
+
+      step2 = Step.new tmpfile.step2 do 
+        step1 = dependencies.first
+        step1.inputs.first + " has " + step1.load.to_s + " characters"
+      end
+
+      step2.dependencies = [step1]
+
+      assert_equal "12 has 2 characters", step2.run
+      assert_equal "12 has 2 characters", step2.run
+    end
+  end
+end
+

data/test/scout/workflow/task/test_inputs.rb

@@ -0,0 +1,182 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/workflow/util'
+
+class TestTaskInput < Test::Unit::TestCase
+  def example_workflow
+    Workflow.annonymous_workflow do
+      input :string, :string, "", "String"
+      input :integer, :integer, "", 1
+      input :float, :float, "", 1.1
+      input :boolean, :boolean, "", false
+      input :array, :array, "", %w(a b)
+      input :path, :path, "", "dir/subdir/file"
+      input :file, :file, ""
+      input :integer_array, :integer_array, "", [1, 2]
+      input :float_array, :float_array, "", [1.1, 2.2]
+      input :boolean_array, :boolean_array, "", [true, false, true]
+      input :path_array, :path_array, "", %w(dir/subdir/file1 dir/subdir/file2)
+      input :file_array, :file_array
+      task :task => :array do
+        inputs
+      end
+    end
+  end
+
+  def example_task
+    example_workflow.tasks[:task]
+  end
+
+  def test_assign_inputs
+    task = self.example_task
+
+    assert_equal [:integer], task.assign_inputs(:string => "String", :integer => 2, :integer_array => %w(1 2)).last
+    
+    TmpFile.with_file("1\n2") do |integer_array_file|
+      assert_equal [:integer], task.assign_inputs(:string => "String", :integer => 2, :integer_array => integer_array_file).last
+    end
+  end
+
+  def test_boolean
+    task = self.example_task
+
+    assert_equal [:boolean], task.assign_inputs(:string => "String", :integer => 1, :boolean => true).last
+
+    TmpFile.with_file("1\n2") do |integer_array_file|
+      Open.open(integer_array_file) do |f|
+        inputs, _ = task.assign_inputs(:string => "String", :integer => 2, :integer_array => f)
+        input_pos = task.inputs.index{|p| p.first == :integer_array}
+        assert File === inputs[input_pos]
+      end
+    end
+  end
+
+  def test_keep_stream
+    task = self.example_task
+
+    assert_equal [:integer], task.assign_inputs(:string => "String", :integer => 2, :integer_array => %w(1 2)).last
+
+    TmpFile.with_file("1\n2") do |integer_array_file|
+      Open.open(integer_array_file) do |f|
+        inputs, _ = task.assign_inputs(:string => "String", :integer => 2, :integer_array => f)
+        input_pos = task.inputs.index{|p| p.first == :integer_array}
+        assert File === inputs[input_pos]
+      end
+    end
+  end
+
+
+  def test_digest_inputs
+    task = self.example_task
+
+    TmpFile.with_file("2\n3") do |integer_array_file|
+      assert_equal task.digest_inputs(:string => "String", :integer => 2, :integer_array => %w(2 3)),
+        task.digest_inputs(:string => "String", :integer => 2, :integer_array => integer_array_file)
+    end
+  end
+
+  def test_digest_stream
+    task = self.example_task
+
+    assert_equal [:integer], task.assign_inputs(:string => "String", :integer => 2, :integer_array => %w(1 2)).last
+
+    TmpFile.with_file("1\n2") do |integer_array_file|
+      hash1 = Open.open(integer_array_file) do |f|
+        task.digest_inputs(:string => "String", :integer => 2, :integer_array => f)
+      end
+      hash2 = Open.open(integer_array_file) do |f|
+        task.digest_inputs(:string => "String", :integer => 2, :integer_array => f)
+      end
+      assert_equal hash1, hash2
+    end
+  end
+
+
+
+  def test_digest_file
+    task = self.example_task
+
+    TmpFile.with_file("2\n3") do |integer_array_file|
+      assert_equal task.digest_inputs(:string => "String", :integer => 2, :integer_array => %w(2 3)),
+        task.digest_inputs(:string => "String", :integer => 2, :integer_array => integer_array_file)
+    end
+  end
+
+  def test_save_and_load
+    task = self.example_task
+
+    TmpFile.with_file("2\n3") do |integer_array_file|
+      inputs = {:string => "String", :integer => 2, :integer_array => integer_array_file, :float_array => %w(1.1 2.2)}
+      original_digest =  task.digest_inputs(inputs)
+
+      TmpFile.with_file do |save_directory|
+        task.save_inputs(save_directory, inputs)
+        new_inputs = task.load_inputs(save_directory)
+        new_digest =  task.digest_inputs(new_inputs)
+        assert_equal original_digest, new_digest
+      end
+    end
+  end
+
+  def test_save_and_load_file
+    task = self.example_task
+
+    TmpFile.with_file("TEST") do |somefile|
+      inputs = {:string => "String", :integer => 2, :file => somefile, :float_array => %w(1.1 2.2)}
+      original_digest =  task.digest_inputs(inputs)
+
+      TmpFile.with_file do |save_directory|
+        task.save_inputs(save_directory, inputs)
+        Open.rm somefile
+        new_inputs = task.load_inputs(save_directory)
+        new_digest =  task.digest_inputs(new_inputs)
+        assert_equal original_digest, new_digest
+      end
+    end
+  end
+
+  def test_save_and_load_file_with_copy
+    task = self.example_task
+
+    TmpFile.with_file("TEST") do |somefile|
+      inputs = {:string => "String", :integer => 2, :file => somefile, :float_array => %w(1.1 2.2)}
+      original_digest =  task.digest_inputs(inputs)
+
+      TmpFile.with_file do |save_directory|
+        task.save_inputs(save_directory, inputs)
+        TmpFile.with_file do |copy_directory|
+          Open.cp save_directory, copy_directory
+          Open.rm_rf save_directory
+          new_inputs = task.load_inputs(copy_directory)
+          new_digest =  task.digest_inputs(new_inputs)
+          assert_equal original_digest, new_digest
+        end
+      end
+    end
+  end
+
+
+  def test_save_and_load_file_array
+    task = self.example_task
+
+    TmpFile.with_file do |dir|
+      file1 = File.join(dir, 'subdir1/file')
+      file2 = File.join(dir, 'subdir2/file')
+
+      Open.write(file1, "TEST1")
+      Open.write(file2, "TEST2")
+      inputs = {:string => "String", :integer => 2, :file_array => [file1, file2], :float_array => %w(1.1 2.2)}
+      original_digest =  task.digest_inputs(inputs)
+
+      TmpFile.with_file do |save_directory|
+        task.save_inputs(save_directory, inputs)
+        Open.rm(file1)
+        Open.rm(file2)
+        new_inputs = task.load_inputs(save_directory)
+        new_digest =  task.digest_inputs(new_inputs)
+        assert_equal original_digest, new_digest
+      end
+    end
+  end
+end

data/test/scout/workflow/test_step.rb

@@ -0,0 +1,36 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestWorkflowStep < Test::Unit::TestCase
+
+  def test_step
+    TmpFile.with_file do |tmpfile|
+      step = Step.new tmpfile, ["12"] do |s|
+        s.length
+      end
+      step.type = :integer
+
+      assert_equal 2, step.run
+    end
+  end
+
+  def test_dependency
+    Log.with_severity 0 do
+      tmpfile = tmpdir.test_step
+      step1 = Step.new tmpfile.step1, ["12"] do |s|
+        s.length
+      end
+
+      step2 = Step.new tmpfile.step2 do 
+        step1 = dependencies.first
+        step1.inputs.first + " has " + step1.load.to_s + " characters"
+      end
+
+      step2.dependencies = [step1]
+
+
+      assert_equal "12 has 2 characters", step2.run
+      assert_equal "12 has 2 characters", step2.run
+    end
+  end
+end