scout-gear 8.0.0 → 8.1.0

data/scout-gear.gemspec

@@ -2,17 +2,17 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-gear 8.0.0 ruby lib
+# stub: scout-gear 8.1.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-gear".freeze
-  s.version = "8.0.0"
+  s.version = "8.1.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Miguel Vazquez".freeze]
-  s.date = "2023-05-23"
-  s.description = "Temporary files, logs, etc.".freeze
+  s.date = "2023-06-15"
+  s.description = "Temporary files, logs, path, resources, persistence, workflows, TSV, etc.".freeze
   s.email = "mikisvaz@gmail.com".freeze
   s.executables = ["scout".freeze]
   s.extra_rdoc_files = [
@@ -28,6 +28,8 @@ Gem::Specification.new do |s|
     "Rakefile",
     "VERSION",
     "bin/scout",
+    "doc/lib/scout/path.md",
+    "doc/lib/scout/workflow/task.md",
     "lib/rbbt-scout.rb",
     "lib/scout-gear.rb",
     "lib/scout.rb",
@@ -56,6 +58,10 @@ Gem::Specification.new do |s|
     "lib/scout/misc/monitor.rb",
     "lib/scout/misc/system.rb",
     "lib/scout/named_array.rb",
+    "lib/scout/offsite.rb",
+    "lib/scout/offsite/ssh.rb",
+    "lib/scout/offsite/step.rb",
+    "lib/scout/offsite/sync.rb",
     "lib/scout/open.rb",
     "lib/scout/open/lock.rb",
     "lib/scout/open/lock/lockfile.rb",
@@ -98,6 +104,7 @@ Gem::Specification.new do |s|
     "lib/scout/tsv/persist/fix_width_table.rb",
     "lib/scout/tsv/persist/serialize.rb",
     "lib/scout/tsv/persist/tokyocabinet.rb",
+    "lib/scout/tsv/stream.rb",
     "lib/scout/tsv/transformer.rb",
     "lib/scout/tsv/traverse.rb",
     "lib/scout/tsv/util.rb",
@@ -111,6 +118,8 @@ Gem::Specification.new do |s|
     "lib/scout/work_queue/worker.rb",
     "lib/scout/workflow.rb",
     "lib/scout/workflow/definition.rb",
+    "lib/scout/workflow/deployment.rb",
+    "lib/scout/workflow/deployment/orchestrator.rb",
     "lib/scout/workflow/documentation.rb",
     "lib/scout/workflow/step.rb",
     "lib/scout/workflow/step/config.rb",
@@ -122,19 +131,23 @@ Gem::Specification.new do |s|
     "lib/scout/workflow/step/provenance.rb",
     "lib/scout/workflow/step/status.rb",
     "lib/scout/workflow/task.rb",
+    "lib/scout/workflow/task/dependencies.rb",
     "lib/scout/workflow/task/inputs.rb",
     "lib/scout/workflow/usage.rb",
     "lib/scout/workflow/util.rb",
     "lib/workflow-scout.rb",
     "scout-gear.gemspec",
     "scout_commands/alias",
+    "scout_commands/doc",
     "scout_commands/find",
     "scout_commands/glob",
+    "scout_commands/offsite",
     "scout_commands/rbbt",
+    "scout_commands/update",
     "scout_commands/workflow/info",
+    "scout_commands/workflow/install",
     "scout_commands/workflow/list",
     "scout_commands/workflow/task",
-    "scout_commands/workflow/task_old",
     "share/color/color_names",
     "share/color/diverging_colors.hex",
     "share/software/install_helpers",
@@ -146,6 +159,10 @@ Gem::Specification.new do |s|
     "test/scout/misc/test_filesystem.rb",
     "test/scout/misc/test_insist.rb",
     "test/scout/misc/test_system.rb",
+    "test/scout/offsite/test_ssh.rb",
+    "test/scout/offsite/test_step.rb",
+    "test/scout/offsite/test_sync.rb",
+    "test/scout/offsite/test_task.rb",
     "test/scout/open/test_lock.rb",
     "test/scout/open/test_remote.rb",
     "test/scout/open/test_stream.rb",
@@ -190,6 +207,7 @@ Gem::Specification.new do |s|
     "test/scout/tsv/test_open.rb",
     "test/scout/tsv/test_parser.rb",
     "test/scout/tsv/test_persist.rb",
+    "test/scout/tsv/test_stream.rb",
     "test/scout/tsv/test_transformer.rb",
     "test/scout/tsv/test_traverse.rb",
     "test/scout/tsv/test_util.rb",
@@ -200,11 +218,13 @@ Gem::Specification.new do |s|
     "test/scout/tsv/util/test_unzip.rb",
     "test/scout/work_queue/test_socket.rb",
     "test/scout/work_queue/test_worker.rb",
+    "test/scout/workflow/deployment/test_orchestrator.rb",
     "test/scout/workflow/step/test_dependencies.rb",
     "test/scout/workflow/step/test_info.rb",
     "test/scout/workflow/step/test_load.rb",
     "test/scout/workflow/step/test_provenance.rb",
     "test/scout/workflow/step/test_status.rb",
+    "test/scout/workflow/task/test_dependencies.rb",
     "test/scout/workflow/task/test_inputs.rb",
     "test/scout/workflow/test_definition.rb",
     "test/scout/workflow/test_documentation.rb",
@@ -213,16 +233,21 @@ Gem::Specification.new do |s|
     "test/scout/workflow/test_usage.rb",
     "test/scout/workflow/test_util.rb",
     "test/test_helper.rb",
-    "test/test_scout-gear.rb"
+    "test/test_scout-gear.rb",
+    "test/test_scout.rb"
   ]
   s.homepage = "http://github.com/mikisvaz/scout-gear".freeze
   s.licenses = ["MIT".freeze]
-  s.rubygems_version = "3.4.11".freeze
+  s.rubygems_version = "3.4.13".freeze
   s.summary = "basic gear for scouts".freeze
 
   s.specification_version = 4
 
   s.add_runtime_dependency(%q<term-ansicolor>.freeze, [">= 0"])
+  s.add_runtime_dependency(%q<net-ssh>.freeze, [">= 0"])
+  s.add_runtime_dependency(%q<matrix>.freeze, [">= 0"])
+  s.add_runtime_dependency(%q<sys-proctable>.freeze, [">= 0"])
+  s.add_runtime_dependency(%q<RubyInline>.freeze, [">= 0"])
   s.add_development_dependency(%q<rdoc>.freeze, ["~> 3.12"])
   s.add_development_dependency(%q<bundler>.freeze, ["~> 1.0"])
   s.add_development_dependency(%q<juwelier>.freeze, ["~> 2.1.0"])

data/scout_commands/doc

@@ -0,0 +1,37 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Print documentation for a module
+
+$ #{$0} [<options>] [module]
+
+If you don't specify the module it prints those available
+
+-h--help Print this help
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+
+module_name = ARGV.first
+if module_name.nil?
+  puts Rbbt.doc.lib.scout.glob("**/*.md").collect{|f| File.basename(f.unset_extension) } * "\n"
+else
+  begin
+    puts Rbbt.doc.lib.scout[module_name].find_with_extension('md', :lib).read
+  rescue
+    puts Rbbt.doc.lib.scout.glob("**/*.md").select{|f| File.basename(f.unset_extension) == module_name }.first.read
+  end
+end
+

data/scout_commands/find

@@ -1,6 +1,7 @@
 #!/usr/bin/env ruby
 
 require 'scout-gear'
+require 'scout/offsite'
 
 $0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
 

data/scout_commands/offsite

@@ -0,0 +1,30 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+require 'scout/offsite'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Description of the tool
+
+$ #{$0} [<options>] <where> <command> ... -- [<protected options>]
+
+Run a command offsite. Use -- to make sure options reach the offsite
+
+-h--help Print this help
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+where = ARGV.shift
+command = ARGV.shift
+
+puts SSHLine.command(where, command, ARGV)

data/scout_commands/update

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Update scout installation
+
+$ #{$0} [<options>] [gem]
+
+If no gem is specified 'scout-gear' is selected.
+
+-h--help Print this help
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+gem_name = ARGV.first
+Misc.update_git(gem_name)
+
+

data/scout_commands/workflow/info

@@ -23,7 +23,19 @@ end
 
 path = ARGV.first
 raise MissingParameterException.new :step_path if path.nil?
-step = Step.load(path)
-
-puts step.info.to_json
+step = Step.load(path.dup)
+
+step.info.each do |k,v|
+  case v
+  when nil
+    next
+  when Exception
+    puts Log.color(:title, "Exception")
+    Log.exception v
+  else
+    puts [Log.color(:title, k), Log.fingerprint(v)] * " = "
+  end
+rescue
+  next
+end
 

data/scout_commands/workflow/install

@@ -0,0 +1,102 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Description of the tool
+
+$ #{$0} [<options>] <filename> [<other|->]*
+
+Install a workflow, or a coma separated list of workflows, from its source. If
+repo is not specified then 'http://github.com/Rbbt-Workflows/' will be used. If
+the workflow is already installed it will be updated. If 'all' is specified
+instead of a particular workflow, all installed workflows will be updated.
+
+You can set the environment variable SCOUT_WORKFLOW_AUTOINSTALL=true to make
+workflows install automatically as required.
+
+-h--help Print this help
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+
+workflow = ARGV[0]
+base_repo = ARGV[1]
+
+raise "No workflow specified" if workflow.nil?
+
+workflow_dir = Scout.etc.workflow_dir.exists? ? Path.setup(Scout.etc.workflow_dir.read.strip) : Scout.workflows.find(:user)
+
+FileUtils.mkdir_p workflow_dir unless File.exist? workflow_dir
+
+all_workflows = workflow_dir.find.glob("*/.git").collect{|d| File.basename(File.dirname(d))}
+workflows = workflow == 'all' ? all_workflows : workflow.split(",")
+
+Misc.in_dir(workflow_dir) do
+  workflows.each do |workflow|
+    case
+    when File.exist?(Misc.snake_case(workflow))
+      Log.info "Updating: " + workflow
+      Misc.in_dir(Misc.snake_case(workflow)) do
+        `git pull`
+        `git submodule init`
+        `git submodule update`
+      end
+    when File.exist?(workflow)
+      Misc.in_dir(workflow) do
+        Log.info "Updating: " + workflow
+        `git pull`
+        `git submodule init`
+        `git submodule update`
+      end
+    else
+      Log.info "Installing: " + workflow
+
+      if base_repo.nil?
+        repo_base_url = Rbbt.etc.workflow_repo.exists? ? Rbbt.etc.workflow_repo.read.strip : 'https://github.com/Rbbt-Workflows/'
+      else
+        repo_base_url = base_repo
+      end
+
+
+      if repo_base_url.include?(workflow) or repo_base_url.include?(Misc.snake_case(workflow))
+        repo = repo_base_url
+      else
+        begin
+          repo = File.join(repo_base_url, workflow + '.git')
+          CMD.cmd("wget '#{repo}' -O /dev/null").read
+        rescue
+          Log.debug "Workflow repo does not exist, trying snake_case: #{ repo }"
+          begin
+            repo = File.join(repo_base_url, Misc.snake_case(workflow) + '.git')
+            CMD.cmd("wget '#{repo}' -O /dev/null").read
+          rescue
+            raise "Workflow repo does not exist: #{ repo }"
+          end
+        end
+      end
+
+      Log.warn "Cloning #{ repo }"
+      Misc.insist do
+        `git clone "#{repo}" #{ Misc.snake_case(workflow) }`
+        raise unless $?.success?
+      end
+      Log.warn "Initializing and updating submodules for #{repo}. You might be prompted for passwords."
+      Misc.in_dir(Misc.snake_case(workflow)) do
+        `git submodule init`
+        `git submodule update`
+      end
+    end
+  end
+end
+

data/scout_commands/workflow/task

@@ -5,7 +5,9 @@ require 'scout'
 $0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
 
 ENV["SCOUT_NO_STREAM"] = "true" if ARGV.include? "--nostream"
+ARGV.delete '--nostream'
 ENV["SCOUT_UPDATE"] = "true" if ARGV.include? "--update"
+ARGV.delete '--update'
 
 options = SOPT.setup <<EOF
 
@@ -17,11 +19,13 @@ $ #{$0} [<options>] <workflow> <task>
 --nostream Disable job streaming
 --update Update jobs with newer dependencies
 -jn--job_name* Name to use as job identifier
+-li--load_inputs* Directory with inputs files to load
 -pf--print_filepath Print the file path
 -prov--provenance Print the step provenance
 -cl--clean Clean the last step
 -rcl--recursive_clean Clean all steps
 -ct--clean_task* Clean a particular task
+-d--deploy* Deploy mode: serial, local, or SLURM (default 'serial')
 EOF
 
 workflow_name, task_name = ARGV
@@ -34,8 +38,9 @@ task = workflow.tasks[task_name.to_sym] if task_name
 
 options[:help] = true if task.nil?
 
-help, provenance, clean, recursive_clean, clean_task, jobname, print_filepath = IndiferentHash.process_options options, 
-  :help, :provenance, :clean, :recursive_clean, :clean_task, :job_name, :print_filepath
+help, provenance, clean, recursive_clean, clean_task, load_inputs, jobname, print_filepath, deploy = IndiferentHash.process_options options, 
+  :help, :provenance, :clean, :recursive_clean, :clean_task, :load_inputs, :job_name, :print_filepath, :deploy,
+  :deploy => 'serial'
 
 if help
   if defined? scout_usage
@@ -48,8 +53,13 @@ if help
   exit 0
 end
 
-job_options = task.get_SOPT(task)
-job = workflow.job(task_name, jobname, job_options)
+if load_inputs
+  job_inputs = workflow.tasks[task_name].load_inputs(load_inputs)
+else
+  job_inputs = task.get_SOPT
+end
+
+job = workflow.job(task_name, jobname, job_inputs)
 
 job.recursive_clean if recursive_clean
 job.clean if clean
@@ -75,7 +85,18 @@ end
 if provenance
   puts Step.prov_report(job)
 else
-  job.run(true) unless job.done?
+  case deploy
+  when "serial"
+    job.run(true) 
+  when "local"
+    orchestrator = Workflow::Orchestrator.new 3, "cpus" => Misc.processors
+    orchestrator.process({}, job)
+  when "slurm"
+    SLURM.orchestrate(job)
+  else 
+    OffsiteStep.setup(job, server: deploy, provided_inputs: job_inputs)
+    job.run
+  end unless job.done?
 
   if print_filepath
     job.join

data/test/scout/offsite/test_ssh.rb

@@ -0,0 +1,15 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestSSH < Test::Unit::TestCase
+  def test_marshal
+    return unless SSHLine.reach?
+
+    assert TrueClass === SSHLine.rbbt(:default, 'true')
+  end
+
+  def test_localhost
+    assert SSHLine.scout('localhost', 'true')
+  end
+end
+

data/test/scout/offsite/test_step.rb

@@ -0,0 +1,33 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestOffsiteStep < Test::Unit::TestCase
+  def test_offsite_task
+    workflow_code =<<-EOF
+module TestWF
+  extend Workflow
+
+  input :string, :string, "String", "string"
+  task :string => :string do |string| string end
+end
+
+TestWF.directory = Path.setup("#{tmpdir.offsite.TestWF}")
+    EOF
+
+    TmpFile.with_file workflow_code, :extension => 'rb' do |wffile|
+      wf = Workflow.require_workflow wffile
+
+      job = wf.job(:string)
+
+      sss 0
+      off = OffsiteStep.setup job, server: 'localhost', workflow_name: wffile
+
+      refute off.done?
+      assert_equal 'string', off.run
+
+      assert off.done?
+      assert_equal 'string', off.run
+    end
+  end
+end
+

data/test/scout/offsite/test_sync.rb

@@ -0,0 +1,36 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestSync < Test::Unit::TestCase
+  def test_sync
+    TmpFile.with_path do |tmpdir|
+      tmpdir = Scout.tmp.tmpdir_sync
+      tmpdir.dir1.foo.write("FOO")
+      tmpdir.dir1.bar.write("BAR")
+
+      TmpFile.with_path do |tmpdir2|
+        Misc.in_dir tmpdir2 do
+          SSHLine.sync([tmpdir.dir1], map: :current)
+
+          assert tmpdir2.glob("**/*").select{|f| f.include?('foo') }.any?
+        end
+      end
+    end
+  end
+
+  def test_sync_dir_map
+    TmpFile.with_path do |tmpdir|
+      tmpdir = Scout.tmp.tmpdir_sync
+      tmpdir.dir1.foo.write("FOO")
+      tmpdir.dir1.bar.write("BAR")
+
+      TmpFile.with_path do |tmpdir2|
+        SSHLine.sync([tmpdir.dir1], map: tmpdir2)
+        Misc.in_dir tmpdir2 do
+          assert tmpdir2.glob("**/*").select{|f| f.include?('foo') }.any?
+        end
+      end
+    end
+  end
+end
+

data/test/scout/resource/test_path.rb

@@ -36,5 +36,11 @@ class TestResourcePath < Test::Unit::TestCase
       assert_include tmpfile.foo.bar.read, "TEST"
     end
   end
+
+  def test_identify
+    p = Scout.data.file.find(:lib)
+    assert p.located?
+    assert_equal "data/file", p.identify
+  end
 end
 

data/test/scout/test_named_array.rb

@@ -33,5 +33,11 @@ ValueB (Entity type)
     NamedArray.add_zipped a, [%w(d), %w(1)]
     assert_equal [%w(a b c d), %w(1 1 1 1)], a
   end
+
+  def test_method_missing
+    a = NamedArray.setup([1,2], [:a, :b])
+    assert_equal 1, a.a
+    assert_equal 2, a.b
+  end
 end
 

data/test/scout/test_persist.rb

@@ -114,6 +114,7 @@ class TestPersist < Test::Unit::TestCase
               end
             end
           end
+
           if IO === io
             Open.consume_stream(io, false)
           else
@@ -137,8 +138,8 @@ class TestPersist < Test::Unit::TestCase
             Open.write(output2, io)
           end
         end
-        Process.wait
-        Process.wait
+        Process.wait pid1
+        Process.wait pid2
 
         assert File.exist?(output1) || File.exist?(output2)
         [pid1, pid2].zip([output2, output1]).each do |pid, found|

data/test/scout/test_tsv.rb

@@ -290,4 +290,21 @@ row2   b  bbb bbbb bb
     tsv = TSV.str_setup("ID~ValueA,ValueB#:type=:flat", {})
     assert_equal "ID", tsv.key_field
   end
+
+  def test_cast_in_header
+    content =<<-EOF
+#: :sep=/\\s+/#:type=:single
+#Id Value
+a 1
+b 2
+c 3
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :key_field => "Value", :grep => "#\\|2")
+      refute tsv.to_s.include?(":cast=:to_f")
+      tsv.cast = :to_f
+      assert_include tsv.to_s, ":cast=:to_f"
+    end
+  end
 end