scout-gear 8.0.0 → 9.0.0

data/scout_commands/workflow/install

@@ -0,0 +1,105 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Description of the tool
+
+$ #{$0} [<options>] <filename> [<other|->]*
+
+Install a workflow, or a coma separated list of workflows, from its source. If
+repo is not specified then 'http://github.com/Rbbt-Workflows/' will be used. If
+the workflow is already installed it will be updated. If 'all' is specified
+instead of a particular workflow, all installed workflows will be updated.
+
+You can set the environment variable SCOUT_WORKFLOW_AUTOINSTALL=true to make
+workflows install automatically as required.
+
+-h--help Print this help
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+
+workflow = ARGV[0]
+base_repo = ARGV[1]
+
+raise "No workflow specified" if workflow.nil?
+
+workflow_dir = Scout.etc.workflow_dir.exists? ? Path.setup(Scout.etc.workflow_dir.read.strip) : Scout.workflows.find(:user)
+
+FileUtils.mkdir_p workflow_dir unless File.exist? workflow_dir
+
+all_workflows = workflow_dir.find.glob("*/.git").collect{|d| File.basename(File.dirname(d))}
+workflows = workflow == 'all' ? all_workflows : workflow.split(",")
+
+Misc.in_dir(workflow_dir) do
+  workflows.each do |workflow|
+    case
+    when File.exist?(Misc.snake_case(workflow))
+      Log.info "Updating: " + workflow
+      Misc.in_dir(Misc.snake_case(workflow)) do
+        `git pull`
+        `git checkout scout 2> /dev/null`
+        `git submodule init`
+        `git submodule update`
+      end
+    when File.exist?(workflow)
+      Misc.in_dir(workflow) do
+        Log.info "Updating: " + workflow
+        `git pull`
+        `git checkout scout 2> /dev/null`
+        `git submodule init`
+        `git submodule update`
+      end
+    else
+      Log.info "Installing: " + workflow
+
+      if base_repo.nil?
+        repo_base_url = Rbbt.etc.workflow_repo.exists? ? Rbbt.etc.workflow_repo.read.strip : 'https://github.com/Rbbt-Workflows/'
+      else
+        repo_base_url = base_repo
+      end
+
+
+      if repo_base_url.include?(workflow) or repo_base_url.include?(Misc.snake_case(workflow))
+        repo = repo_base_url
+      else
+        begin
+          repo = File.join(repo_base_url, workflow + '.git')
+          CMD.cmd("wget '#{repo}' -O /dev/null").read
+        rescue
+          Log.debug "Workflow repo does not exist, trying snake_case: #{ repo }"
+          begin
+            repo = File.join(repo_base_url, Misc.snake_case(workflow) + '.git')
+            CMD.cmd("wget '#{repo}' -O /dev/null").read
+          rescue
+            raise "Workflow repo does not exist: #{ repo }"
+          end
+        end
+      end
+
+      Log.warn "Cloning #{ repo }"
+      Misc.insist do
+        `git clone "#{repo}" #{ Misc.snake_case(workflow) }`
+        raise unless $?.success?
+      end
+      Log.warn "Initializing and updating submodules for #{repo}. You might be prompted for passwords."
+      Misc.in_dir(Misc.snake_case(workflow)) do
+        `git checkout scout 2> /dev/null`
+        `git submodule init`
+        `git submodule update`
+      end
+    end
+  end
+end
+

data/scout_commands/workflow/task

@@ -5,7 +5,9 @@ require 'scout'
 $0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
 
 ENV["SCOUT_NO_STREAM"] = "true" if ARGV.include? "--nostream"
+ARGV.delete '--nostream'
 ENV["SCOUT_UPDATE"] = "true" if ARGV.include? "--update"
+ARGV.delete '--update'
 
 options = SOPT.setup <<EOF
 
@@ -16,12 +18,15 @@ $ #{$0} [<options>] <workflow> <task>
 -h--help Print this help
 --nostream Disable job streaming
 --update Update jobs with newer dependencies
--jn--job_name* Name to use as job identifier
+-jn--jobname* Name to use as job identifier
+-li--load_inputs* Directory with inputs files to load
 -pf--print_filepath Print the file path
 -prov--provenance Print the step provenance
 -cl--clean Clean the last step
 -rcl--recursive_clean Clean all steps
 -ct--clean_task* Clean a particular task
+-d--deploy* Deploy mode: serial, local, or SLURM (default 'serial')
+-od--override_deps* Override deps using 'Workflow#task=<path>' array_separated
 EOF
 
 workflow_name, task_name = ARGV
@@ -34,8 +39,9 @@ task = workflow.tasks[task_name.to_sym] if task_name
 
 options[:help] = true if task.nil?
 
-help, provenance, clean, recursive_clean, clean_task, jobname, print_filepath = IndiferentHash.process_options options, 
-  :help, :provenance, :clean, :recursive_clean, :clean_task, :job_name, :print_filepath
+help, provenance, clean, recursive_clean, clean_task, load_inputs, jobname, print_filepath, deploy, override_deps = IndiferentHash.process_options options, 
+  :help, :provenance, :clean, :recursive_clean, :clean_task, :load_inputs, :jobname, :print_filepath, :deploy, :override_deps,
+  :deploy => 'serial'
 
 if help
   if defined? scout_usage
@@ -48,8 +54,20 @@ if help
   exit 0
 end
 
-job_options = task.get_SOPT(task)
-job = workflow.job(task_name, jobname, job_options)
+if load_inputs
+  job_inputs = workflow.tasks[task_name].load_inputs(load_inputs)
+else
+  job_inputs = task.get_SOPT
+end
+
+if override_deps
+  override_deps.split($array_separator || ",").each do |part|
+    t_, value = part.split("=")
+    job_inputs.merge!( t_ => value)
+  end
+end
+
+job = workflow.job(task_name, jobname, job_inputs)
 
 job.recursive_clean if recursive_clean
 job.clean if clean
@@ -69,13 +87,35 @@ if clean_task
     end
 
     job.clean if job.task_name.to_s == clean_task.to_s
+    job.clean unless job.updated?
   end
 end
 
 if provenance
   puts Step.prov_report(job)
 else
-  job.run(true) unless job.done?
+  case deploy
+  when "serial"
+    job.run(true) 
+  when "local"
+    orchestrator = Workflow::Orchestrator.new 3, "cpus" => Misc.processors
+    orchestrator.process({}, job)
+  when "slurm"
+    require 'rbbt-scout'
+    require_relative '../../modules/rbbt-util/lib/rbbt/hpc'
+    HPC::BATCH_MODULE = HPC.batch_system "SLURM"
+    HPC::BATCH_MODULE.orchestrate_job(job, {})
+    job.grace
+  else 
+    if deploy.end_with?('-slurm')
+      server = deploy.sub('-slurm','')
+      OffsiteStep.setup(job, server: server, slurm: true)
+    else
+      OffsiteStep.setup(job, server: deploy)
+    end
+
+    job.run
+  end unless job.done?
 
   if print_filepath
     job.join

data/share/software/install_helpers

@@ -32,6 +32,7 @@ else
 		local basedir="$(realpath $1)"
 		local path="$(realpath "$2")"
     local pre=""
+    basedir=${basedir%.}
     while [[ ! $path = $basedir* ]]; do
       pre=${pre}../
       basedir=$(dirname $basedir)
@@ -44,9 +45,8 @@ function link(){
   local source="$1"
   local target="$2"
   local rel_source=$(relative_path "$(dirname $target)" "$source")
-  
+
   [ -h "$target" ] && rm "$target"
-  echo ln -s "$rel_source" "$target" 1>&2
   [ -h "$target" ] || ln -s "$rel_source" "$target"
 }
 

data/share/templates/command

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Description of the tool
+
+$ #{$0} [<options>] <filename> [<other|->]*
+
+-h--help Print this help
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+
+

data/share/templates/workflow.rb

@@ -0,0 +1,14 @@
+Misc.add_libdir if __FILE__ == $0
+
+#require 'scout/sources/MODULE'
+
+module MODULE
+  extend Workflow
+
+
+end
+
+#require 'MODULE/tasks/basic.rb'
+
+#require 'scout/knowledge_base/MODULE'
+#require 'scout/entity/MODULE'

data/test/scout/offsite/test_ssh.rb

@@ -0,0 +1,15 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestSSH < Test::Unit::TestCase
+  def test_marshal
+    return unless SSHLine.reach?
+
+    assert TrueClass === SSHLine.rbbt(:default, 'true')
+  end
+
+  def test_localhost
+    assert SSHLine.scout('localhost', 'true')
+  end
+end
+

data/test/scout/offsite/test_step.rb

@@ -0,0 +1,32 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestOffsiteStep < Test::Unit::TestCase
+  def test_offsite_task
+    workflow_code =<<-EOF
+module TestWF
+  extend Workflow
+
+  input :string, :string, "String", "string"
+  task :string => :string do |string| string end
+end
+
+TestWF.directory = Path.setup("#{tmpdir.offsite.TestWF}")
+    EOF
+
+    TmpFile.with_file workflow_code, :extension => 'rb' do |wffile|
+      wf = Workflow.require_workflow wffile
+
+      job = wf.job(:string)
+
+      off = OffsiteStep.setup job, server: 'localhost', workflow_name: wffile
+
+      refute off.done?
+      assert_equal 'string', off.run
+
+      assert off.done?
+      assert_equal 'string', off.run
+    end
+  end
+end
+

data/test/scout/offsite/test_sync.rb

@@ -0,0 +1,36 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestSync < Test::Unit::TestCase
+  def test_sync
+    TmpFile.with_path do |tmpdir|
+      tmpdir = Scout.tmp.tmpdir_sync
+      tmpdir.dir1.foo.write("FOO")
+      tmpdir.dir1.bar.write("BAR")
+
+      TmpFile.with_path do |tmpdir2|
+        Misc.in_dir tmpdir2 do
+          SSHLine.sync([tmpdir.dir1], map: :current)
+
+          assert tmpdir2.glob("**/*").select{|f| f.include?('foo') }.any?
+        end
+      end
+    end
+  end
+
+  def test_sync_dir_map
+    TmpFile.with_path do |tmpdir|
+      tmpdir = Scout.tmp.tmpdir_sync
+      tmpdir.dir1.foo.write("FOO")
+      tmpdir.dir1.bar.write("BAR")
+
+      TmpFile.with_path do |tmpdir2|
+        SSHLine.sync([tmpdir.dir1], map: tmpdir2)
+        Misc.in_dir tmpdir2 do
+          assert tmpdir2.glob("**/*").select{|f| f.include?('foo') }.any?
+        end
+      end
+    end
+  end
+end
+

data/test/scout/resource/test_path.rb

@@ -36,5 +36,11 @@ class TestResourcePath < Test::Unit::TestCase
       assert_include tmpfile.foo.bar.read, "TEST"
     end
   end
+
+  def test_identify
+    p = Scout.data.file.find(:lib)
+    assert p.located?
+    assert_equal "data/file", p.identify
+  end
 end
 

data/test/scout/test_named_array.rb

@@ -33,5 +33,11 @@ ValueB (Entity type)
     NamedArray.add_zipped a, [%w(d), %w(1)]
     assert_equal [%w(a b c d), %w(1 1 1 1)], a
   end
+
+  def test_method_missing
+    a = NamedArray.setup([1,2], [:a, :b])
+    assert_equal 1, a.a
+    assert_equal 2, a.b
+  end
 end
 

data/test/scout/test_persist.rb

@@ -114,6 +114,7 @@ class TestPersist < Test::Unit::TestCase
               end
             end
           end
+
           if IO === io
             Open.consume_stream(io, false)
           else
@@ -137,8 +138,8 @@ class TestPersist < Test::Unit::TestCase
             Open.write(output2, io)
           end
         end
-        Process.wait
-        Process.wait
+        Process.waitpid pid1
+        Process.waitpid pid2
 
         assert File.exist?(output1) || File.exist?(output2)
         [pid1, pid2].zip([output2, output1]).each do |pid, found|

data/test/scout/test_tsv.rb

@@ -290,4 +290,21 @@ row2   b  bbb bbbb bb
     tsv = TSV.str_setup("ID~ValueA,ValueB#:type=:flat", {})
     assert_equal "ID", tsv.key_field
   end
+
+  def test_cast_in_header
+    content =<<-EOF
+#: :sep=/\\s+/#:type=:single
+#Id Value
+a 1
+b 2
+c 3
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :key_field => "Value", :grep => "#\\|2")
+      refute tsv.to_s.include?(":cast=:to_f")
+      tsv.cast = :to_f
+      assert_include tsv.to_s, ":cast=:to_f"
+    end
+  end
 end

data/test/scout/test_work_queue.rb

@@ -56,7 +56,7 @@ class TestWorkQueue < Test::Unit::TestCase
       end
     end
 
-    Process.wait pid
+    Process.waitpid pid
     q.close
 
     q.join
@@ -90,60 +90,82 @@ class TestWorkQueue < Test::Unit::TestCase
   end
 
   def test_queue_error
-    num = 100
-    reps = 10_000
-
-    q = WorkQueue.new num do |obj|
-      raise ScoutException if rand < 0.1
-      [Process.pid.to_s, obj.to_s] * " "
-    end
-
-    res = []
-    q.process do |out|
-      res << out
-    end
+    5.times do |i|
+      num = 100
+      reps = 10_000
 
-    Log.with_severity 7 do
-      pid = Process.fork do
-        reps.times do |i|
-          q.write i
-        end
+      q = WorkQueue.new num do |obj|
+        raise ScoutException if rand < 0.1
+        [Process.pid.to_s, obj.to_s] * " "
       end
 
-      Process.waitpid pid
+      res = []
+      q.process do |out|
+        res << out
+      end
 
-      assert_raise ScoutException do
-        q.join
-        t.join
+      Log.with_severity 7 do
+        t = Thread.new do
+          Thread.current["name"] = "queue writer"
+          Thread.current.report_on_exception = false
+          reps.times do |i|
+            q.write i
+          end
+          q.close
+        end
+        Thread.pass until t["name"]
+
+        assert_raise ScoutException do
+          begin
+            q.join(false)
+          rescue
+            t.raise($!)
+            raise $!
+          ensure
+            t.join
+          end
+        end
       end
     end
   end
 
   def test_queue_error_in_input
-    num = 100
-    reps = 10_000
+    5.times do |i|
+      num = 100
+      reps = 10_000
 
-    q = WorkQueue.new num do |obj|
-      [Process.pid.to_s, obj.to_s] * " "
-    end
-
-    res = []
-    q.process do |out|
-      raise ScoutException 
-      res << out
-    end
+      q = WorkQueue.new num do |obj|
+        [Process.pid.to_s, obj.to_s] * " "
+      end
 
-    Log.with_severity 7 do
-      pid = Process.fork do
-        reps.times do |i|
-          q.write i
-        end
-        q.close
+      res = []
+      q.process do |out|
+        raise ScoutException 
+        res << out
       end
 
-      assert_raise ScoutException do
-        q.join
-        t.join
+      Log.with_severity 7 do
+        t = Thread.new do
+          Thread.current.report_on_exception = false
+          Thread.current["name"] = "queue writer"
+          reps.times do |i|
+            q.write i
+          end
+          q.close
+        end
+        Thread.pass until t["name"]
+
+        assert_raise ScoutException do
+          begin
+            q.join(false)
+          rescue Exception
+            t.raise($!)
+            raise $!
+          ensure
+            t.join
+            q.clean
+          end
+        end
       end
     end
   end

data/test/scout/tsv/persist/test_adapter.rb

@@ -3,7 +3,7 @@ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1
 
 require 'scout/tsv'
 class TestTSVAdapter < Test::Unit::TestCase
-  def _test_get_set
+  def test_get_set
     tsv = TSV.setup({}, :type => :list, :key_field => "Key", :fields => %w(one two three))
     tsv.type = :list
     tsv.extend TSVAdapter

data/test/scout/tsv/test_index.rb

@@ -43,6 +43,20 @@ row2    a    b    id3
     end
   end
 
+  def test_index_from_flat
+    content =<<-'EOF'
+#: :sep=" "#:type=:flat
+#Id    ValueA
+row1    a aa aaa
+row2    b bb bbb
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      index = TSV.index(filename, :target => "Id")
+      assert_equal "row1", index["aa"]
+    end
+  end
+
   def test_from_tsv
     content =<<-'EOF'
 #: :sep=/\s+/#:type=:double#:merge=:concat

data/test/scout/tsv/test_parser.rb

@@ -95,6 +95,27 @@ k a|A b|B
     assert_equal 'a', tsv['k'][0][0]
   end
 
+  def test_parse_head
+    content =<<-EOF
+#: :sep=" "#:type=:double
+#Key ValueA ValueB
+k a|A b|B
+k1 a|A b|B
+k2 a|A b|B
+k3 a|A b|B
+k4 a|A b|B
+    EOF
+    content = StringIO.new content
+
+    tsv = TSV.parse(content, :head => 2)
+    assert_equal 2, tsv.keys.length
+
+    content.rewind
+    tsv = TSV.parse(content, :head => 3)
+    assert_equal 3, tsv.keys.length
+  end
+
+
   def test_parse_fields
     content =<<-EOF
 #: :sep=" "#:type=:double
@@ -119,6 +140,20 @@ k a|A b|B
     assert_equal [%w(b B), %w(k)], tsv['k']
   end
 
+  def test_parse_flat
+    content =<<-EOF
+#: :sep=" "#:type=:flat
+#Key ValueA
+row1 a aa aaa
+row2 b bb bbb
+    EOF
+
+    tsv = TSV.open(content)
+    assert_equal %w(a aa aaa), tsv["row1"]
+    tsv = TSV.open(content, :fields => ["ValueA"])
+    assert_equal %w(a aa aaa), tsv["row1"]
+  end
+
   def test_parse_key
     content =<<-EOF
 #: :sep=" "#:type=:double