scout-gear 7.3.0 → 8.1.0

data/scout_commands/workflow/install

@@ -0,0 +1,102 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Description of the tool
+
+$ #{$0} [<options>] <filename> [<other|->]*
+
+Install a workflow, or a coma separated list of workflows, from its source. If
+repo is not specified then 'http://github.com/Rbbt-Workflows/' will be used. If
+the workflow is already installed it will be updated. If 'all' is specified
+instead of a particular workflow, all installed workflows will be updated.
+
+You can set the environment variable SCOUT_WORKFLOW_AUTOINSTALL=true to make
+workflows install automatically as required.
+
+-h--help Print this help
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+
+workflow = ARGV[0]
+base_repo = ARGV[1]
+
+raise "No workflow specified" if workflow.nil?
+
+workflow_dir = Scout.etc.workflow_dir.exists? ? Path.setup(Scout.etc.workflow_dir.read.strip) : Scout.workflows.find(:user)
+
+FileUtils.mkdir_p workflow_dir unless File.exist? workflow_dir
+
+all_workflows = workflow_dir.find.glob("*/.git").collect{|d| File.basename(File.dirname(d))}
+workflows = workflow == 'all' ? all_workflows : workflow.split(",")
+
+Misc.in_dir(workflow_dir) do
+  workflows.each do |workflow|
+    case
+    when File.exist?(Misc.snake_case(workflow))
+      Log.info "Updating: " + workflow
+      Misc.in_dir(Misc.snake_case(workflow)) do
+        `git pull`
+        `git submodule init`
+        `git submodule update`
+      end
+    when File.exist?(workflow)
+      Misc.in_dir(workflow) do
+        Log.info "Updating: " + workflow
+        `git pull`
+        `git submodule init`
+        `git submodule update`
+      end
+    else
+      Log.info "Installing: " + workflow
+
+      if base_repo.nil?
+        repo_base_url = Rbbt.etc.workflow_repo.exists? ? Rbbt.etc.workflow_repo.read.strip : 'https://github.com/Rbbt-Workflows/'
+      else
+        repo_base_url = base_repo
+      end
+
+
+      if repo_base_url.include?(workflow) or repo_base_url.include?(Misc.snake_case(workflow))
+        repo = repo_base_url
+      else
+        begin
+          repo = File.join(repo_base_url, workflow + '.git')
+          CMD.cmd("wget '#{repo}' -O /dev/null").read
+        rescue
+          Log.debug "Workflow repo does not exist, trying snake_case: #{ repo }"
+          begin
+            repo = File.join(repo_base_url, Misc.snake_case(workflow) + '.git')
+            CMD.cmd("wget '#{repo}' -O /dev/null").read
+          rescue
+            raise "Workflow repo does not exist: #{ repo }"
+          end
+        end
+      end
+
+      Log.warn "Cloning #{ repo }"
+      Misc.insist do
+        `git clone "#{repo}" #{ Misc.snake_case(workflow) }`
+        raise unless $?.success?
+      end
+      Log.warn "Initializing and updating submodules for #{repo}. You might be prompted for passwords."
+      Misc.in_dir(Misc.snake_case(workflow)) do
+        `git submodule init`
+        `git submodule update`
+      end
+    end
+  end
+end
+

data/scout_commands/workflow/task

@@ -4,6 +4,11 @@ require 'scout'
 
 $0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
 
+ENV["SCOUT_NO_STREAM"] = "true" if ARGV.include? "--nostream"
+ARGV.delete '--nostream'
+ENV["SCOUT_UPDATE"] = "true" if ARGV.include? "--update"
+ARGV.delete '--update'
+
 options = SOPT.setup <<EOF
 
 Run a workflow job
@@ -11,11 +16,16 @@ Run a workflow job
 $ #{$0} [<options>] <workflow> <task>
 
 -h--help Print this help
+--nostream Disable job streaming
+--update Update jobs with newer dependencies
 -jn--job_name* Name to use as job identifier
+-li--load_inputs* Directory with inputs files to load
 -pf--print_filepath Print the file path
 -prov--provenance Print the step provenance
 -cl--clean Clean the last step
 -rcl--recursive_clean Clean all steps
+-ct--clean_task* Clean a particular task
+-d--deploy* Deploy mode: serial, local, or SLURM (default 'serial')
 EOF
 
 workflow_name, task_name = ARGV
@@ -28,7 +38,11 @@ task = workflow.tasks[task_name.to_sym] if task_name
 
 options[:help] = true if task.nil?
 
-if options[:help]
+help, provenance, clean, recursive_clean, clean_task, load_inputs, jobname, print_filepath, deploy = IndiferentHash.process_options options, 
+  :help, :provenance, :clean, :recursive_clean, :clean_task, :load_inputs, :job_name, :print_filepath, :deploy,
+  :deploy => 'serial'
+
+if help
   if defined? scout_usage
     scout_usage 
   else
@@ -39,18 +53,52 @@ if options[:help]
   exit 0
 end
 
-job_options = task.get_SOPT(task)
-job = workflow.job(task_name, options[:job_name], job_options)
+if load_inputs
+  job_inputs = workflow.tasks[task_name].load_inputs(load_inputs)
+else
+  job_inputs = task.get_SOPT
+end
 
-job.recursive_clean if options[:recursive_clean]
-job.clean if options[:clean]
+job = workflow.job(task_name, jobname, job_inputs)
+
+job.recursive_clean if recursive_clean
+job.clean if clean
+
+if clean_task 
+  ENV["SCOUT_UPDATE"] = 'true'
+  clean_task.split(",").each do |clean_task|
+    if clean_task.include? "#"
+      clean_workflow, clean_task = clean_task.split("#")
+    end
 
-if options[:provenance]
+    job.rec_dependencies.each do |dep|
+      next unless dep.task_name.to_s == clean_task.to_s
+      next unless clean_workflow.nil? || clean_workflow == dep.workflow.to_s
+      dep.clean 
+      dep.set_info :status, :cleaned
+    end
+
+    job.clean if job.task_name.to_s == clean_task.to_s
+  end
+end
+
+if provenance
   puts Step.prov_report(job)
 else
-  job.run unless job.done?
-
-  if options[:print_filepath]
+  case deploy
+  when "serial"
+    job.run(true) 
+  when "local"
+    orchestrator = Workflow::Orchestrator.new 3, "cpus" => Misc.processors
+    orchestrator.process({}, job)
+  when "slurm"
+    SLURM.orchestrate(job)
+  else 
+    OffsiteStep.setup(job, server: deploy, provided_inputs: job_inputs)
+    job.run
+  end unless job.done?
+
+  if print_filepath
     job.join
     path = job.path
     path = path.find if Path === path

data/test/scout/offsite/test_ssh.rb

@@ -0,0 +1,15 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestSSH < Test::Unit::TestCase
+  def test_marshal
+    return unless SSHLine.reach?
+
+    assert TrueClass === SSHLine.rbbt(:default, 'true')
+  end
+
+  def test_localhost
+    assert SSHLine.scout('localhost', 'true')
+  end
+end
+

data/test/scout/offsite/test_step.rb

@@ -0,0 +1,33 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestOffsiteStep < Test::Unit::TestCase
+  def test_offsite_task
+    workflow_code =<<-EOF
+module TestWF
+  extend Workflow
+
+  input :string, :string, "String", "string"
+  task :string => :string do |string| string end
+end
+
+TestWF.directory = Path.setup("#{tmpdir.offsite.TestWF}")
+    EOF
+
+    TmpFile.with_file workflow_code, :extension => 'rb' do |wffile|
+      wf = Workflow.require_workflow wffile
+
+      job = wf.job(:string)
+
+      sss 0
+      off = OffsiteStep.setup job, server: 'localhost', workflow_name: wffile
+
+      refute off.done?
+      assert_equal 'string', off.run
+
+      assert off.done?
+      assert_equal 'string', off.run
+    end
+  end
+end
+

data/test/scout/offsite/test_sync.rb

@@ -0,0 +1,36 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestSync < Test::Unit::TestCase
+  def test_sync
+    TmpFile.with_path do |tmpdir|
+      tmpdir = Scout.tmp.tmpdir_sync
+      tmpdir.dir1.foo.write("FOO")
+      tmpdir.dir1.bar.write("BAR")
+
+      TmpFile.with_path do |tmpdir2|
+        Misc.in_dir tmpdir2 do
+          SSHLine.sync([tmpdir.dir1], map: :current)
+
+          assert tmpdir2.glob("**/*").select{|f| f.include?('foo') }.any?
+        end
+      end
+    end
+  end
+
+  def test_sync_dir_map
+    TmpFile.with_path do |tmpdir|
+      tmpdir = Scout.tmp.tmpdir_sync
+      tmpdir.dir1.foo.write("FOO")
+      tmpdir.dir1.bar.write("BAR")
+
+      TmpFile.with_path do |tmpdir2|
+        SSHLine.sync([tmpdir.dir1], map: tmpdir2)
+        Misc.in_dir tmpdir2 do
+          assert tmpdir2.glob("**/*").select{|f| f.include?('foo') }.any?
+        end
+      end
+    end
+  end
+end
+

data/test/scout/open/test_stream.rb

@@ -334,63 +334,65 @@ row1 A B C
     assert_equal %w(## ## ## #Row row1 row2 row3), sorted.read.split("\n").collect{|l| l.split(" ").first}
   end
 
-  #  def test_collapse_stream
-  #    text=<<-EOF
-  #row1 A B C
-  #row1 a b c
-  #row2 AA BB CC
-  #row2 aa bb cc
-  #    EOF
-  #
-  #    s = StringIO.new text
-  #    tsv = TSV.open Misc.collapse_stream(s,nil, " "), :sep => " " 
-  #    assert_equal ["A", "a"], tsv["row1"][0]
-  #    assert_equal ["BB", "bb"], tsv["row2"][1]
-  #  end
-  #
-  #  def test_collapse_sum
-  #    text=<<-EOF
-  #row1 12
-  #row1 4
-  #row2 10
-  #row2 6
-  #    EOF
-  #
-  #    s = StringIO.new text
-  #    stream = Misc.collapse_stream(s,nil, " ") do |parts|
-  #      next nil if parts.empty?
-  #      parts.first.split("|").collect{|p| p.to_f}.inject(0){|acc,e| acc += e}.to_s
-  #    end
-  #    tsv = TSV.open  stream, :sep => " " 
-  #    ppp tsv.to_s
-  #  end
-  #
-  #  def test_collapse_stream_gap
-  #    text=<<-EOF
-  #row2 AA BB 
-  #row2 aa bb cc
-  #    EOF
-  #
-  #    s = StringIO.new text
-  #    assert Misc.collapse_stream(s, nil, " ").read =~  /\|cc$/
-  #    
-  #    text=<<-EOF
-  #row2 aa bb cc
-  #row2 AA BB 
-  #    EOF
-  #
-  #    s = StringIO.new text
-  #    assert Misc.collapse_stream(s, nil, " ").read =~  /cc\|$/
-  #
-  #    text=<<-EOF
-  #row2 AA BB
-  #row2 aa bb cc
-  #    EOF
-  #
-  #    s = StringIO.new text
-  #    assert Misc.collapse_stream(s, nil, " ").read =~  /\|cc$/
-  #
-  #  end
+  def test_collapse_stream
+    text=<<-EOF
+row1 A B C
+row1 a b c
+row2 AA BB CC
+row2 aa bb cc
+    EOF
+
+    s = StringIO.new text
+    stream = Open.collapse_stream(s, sep: " ")
+    txt = stream.read
+    assert_include txt, "A|a"
+    assert_include txt, "B|b"
+    assert_include txt, "C|c"
+  end
+
+
+  def test_collapse_sum
+    text=<<-EOF
+row1 12
+row1 4
+row2 10
+row2 6
+    EOF
+
+    s = StringIO.new text
+    stream = Open.collapse_stream(s, sep: " ") do |parts|
+      next nil if parts.empty?
+      parts.first.split("|").collect{|p| p.to_f}.inject(0){|acc,e| acc += e}.to_s
+    end
+    assert_include stream.read, "row1 16"
+  end
+
+
+  def test_collapse_stream_gap
+    text=<<-EOF
+row2 AA BB 
+row2 aa bb cc
+    EOF
+
+    s = StringIO.new text
+    assert Open.collapse_stream(s, sep: " ").read =~  /\|cc$/
+    
+    text=<<-EOF
+row2 aa bb cc
+row2 AA BB 
+    EOF
+
+    s = StringIO.new text
+    assert Open.collapse_stream(s, sep: " ").read =~  /cc\|$/
+
+    text=<<-EOF
+row2 AA BB
+row2 aa bb cc
+    EOF
+
+    s = StringIO.new text
+    assert Open.collapse_stream(s, sep: " ").read =~  /\|cc$/
+  end
   #
   #
   #  def test_paste_stream
@@ -433,7 +435,7 @@ row1 A B C
   #    s = StringIO.new text
   #    sorted = Misc.sort_stream(s)
   #    assert_equal %w(## ## ## #Row row2 row3 row1), text.split("\n").collect{|l| l.split(" ").first}
-#    assert_equal %w(## ## ## #Row row1 row2 row3), sorted.read.split("\n").collect{|l| l.split(" ").first}
+  #    assert_equal %w(## ## ## #Row row1 row2 row3), sorted.read.split("\n").collect{|l| l.split(" ").first}
 #  end
 #
 #  def test_sort_long_stream

data/test/scout/path/test_find.rb

@@ -68,7 +68,6 @@ class TestPathFind < Test::Unit::TestCase
     p = Path.setup("/tmp/foo/bar")
     assert p.located?
     assert_equal_path p, p.find
-    
   end
 
   def test_custom
@@ -106,5 +105,15 @@ class TestPathFind < Test::Unit::TestCase
       assert Open.exist?(tmpdir.somefile)
     end
   end
+
+  def test_with_extension
+    dir = tmpdir.directory[__method__]
+    list = %w(a b)
+    Misc.in_dir(dir) do
+      file = dir.foo
+      Open.write(file.set_extension('list'), list * "\n")
+      assert_equal list, file.list
+    end
+  end
 end
 

data/test/scout/resource/test_path.rb

@@ -36,5 +36,11 @@ class TestResourcePath < Test::Unit::TestCase
       assert_include tmpfile.foo.bar.read, "TEST"
     end
   end
+
+  def test_identify
+    p = Scout.data.file.find(:lib)
+    assert p.located?
+    assert_equal "data/file", p.identify
+  end
 end
 

data/test/scout/resource/test_produce.rb

@@ -59,4 +59,19 @@ end
     assert_include File.open(TestResource.tmp.test.work.footest.bar.find).read, "OTHER"
     assert_include File.open(TestResource.tmp.test.work.footest.foo_bar.find).read, "OTHER"
   end
+
+  def test_produce_with_extension
+    dir = tmpdir.directory[__method__]
+    dir.pkgdir = Scout
+    list = %w(a b)
+
+    Scout.claim dir["foo.list"], :proc do
+      list
+    end
+
+    Misc.in_dir(dir) do
+      file = dir.foo
+      assert_equal list, file.list
+    end
+  end
 end

data/test/scout/test_meta_extension.rb

@@ -8,6 +8,13 @@ class TestMetaExtension < Test::Unit::TestCase
     extension_attr :code, :code2
   end
 
+  module ExtensionClass2
+    extend MetaExtension
+
+    extension_attr :code3, :code4
+  end
+
+
   def test_setup_annotate
     str = "String"
     ExtensionClass.setup(str, :code)
@@ -51,5 +58,23 @@ class TestMetaExtension < Test::Unit::TestCase
     assert o.extension_attr_hash.include?(:code)
     assert o.extension_attr_hash.include?(:code2)
   end
+
+  def test_twice
+    str = "String"
+
+    ExtensionClass.setup(str, :code2 => :code)
+    assert_equal :code, str.code2
+    assert_include str.instance_variable_get(:@extension_attrs), :code
+
+    str.extend ExtensionClass2
+    str.code3 = :code_alt
+    assert_equal :code, str.code2
+    assert_equal :code_alt, str.code3
+    assert_include str.instance_variable_get(:@extension_attrs), :code
+    assert_include str.instance_variable_get(:@extension_attrs), :code3
+
+    assert_include str.extension_attr_hash, :code
+    assert_include str.extension_attr_hash, :code3
+  end
 end
 

data/test/scout/test_named_array.rb

@@ -15,5 +15,29 @@ ValueB (Entity type)
     assert_equal 1, NamedArray.identify_name(names, "ValueB")
     assert_equal 1, NamedArray.identify_name(names, 1)
   end
+
+  def test_missing_field
+    a = NamedArray.setup([1,2], [:a, :b])
+    assert_equal 1, a[:a]
+    assert_equal nil, a[:c]
+  end
+
+  def test_zip_fields
+    a = [%w(a b), %w(1 1)]
+    assert_equal [%w(a 1), %w(b 1)], NamedArray.zip_fields(a)
+  end
+
+  def test_add_zipped
+    a = [%w(a b), %w(1 1)]
+    NamedArray.add_zipped a, [%w(c), %w(1)]
+    NamedArray.add_zipped a, [%w(d), %w(1)]
+    assert_equal [%w(a b c d), %w(1 1 1 1)], a
+  end
+
+  def test_method_missing
+    a = NamedArray.setup([1,2], [:a, :b])
+    assert_equal 1, a.a
+    assert_equal 2, a.b
+  end
 end
 

data/test/scout/test_persist.rb

@@ -114,6 +114,7 @@ class TestPersist < Test::Unit::TestCase
               end
             end
           end
+
           if IO === io
             Open.consume_stream(io, false)
           else
@@ -137,8 +138,8 @@ class TestPersist < Test::Unit::TestCase
             Open.write(output2, io)
           end
         end
-        Process.wait
-        Process.wait
+        Process.wait pid1
+        Process.wait pid2
 
         assert File.exist?(output1) || File.exist?(output2)
         [pid1, pid2].zip([output2, output1]).each do |pid, found|
@@ -156,6 +157,12 @@ class TestPersist < Test::Unit::TestCase
     end
   end
 
+  def test_path_prefix
+    Persist.persist('foo', :tsv, :prefix => "TSV") do |filename|
+      assert File.basename(filename).start_with? "TSV"
+    end
+  end
+
   def __test_speed
     times = 100_000
     TmpFile.with_file do |tmpfile|