scout-gear 8.0.0 → 9.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 48f6bb392f3391e2816e701887bfd98617b1a68bc533c8b6de3307f25c84906d
-  data.tar.gz: 476858f63deaa7fa9fc531c7872bdaefa1d6b3ed32cdacbe41f53d1d170fcc87
+  metadata.gz: 98b3f84350f2f451b717f004bfc97cd7767f7e0fed58e1a2d0cd023bc700185f
+  data.tar.gz: '00847ad4d92b1cab22305ea867dc12c870b22e192e79536917f82aa90cc82989'
 SHA512:
-  metadata.gz: c61501c8cc79b57b64cb8613efc8b00fb84908bf0d55b008c6a6a39b99ea6bffa53630a8e134c07a49cb7bc4145ec181df057b6ba1eebfd2d722b2aa2d2e63d0
-  data.tar.gz: 100356a04cd34eba627239668747023ee5be4455ebbee715df39dbe04e9c8dfcac7007ce310e4622cf0c7e16ee9159e6fc2d4eefcaa486c24519c7f99185facf
+  metadata.gz: 4af5e13873fa13dea33161e9200aa725bb4329899e9242ae6504ffae2ab82e7eb01b82a53edc2a4331ede06d0f12bb07f86f579cb29ecbf0d36e791bf6d53f3e
+  data.tar.gz: fc773eb933e5c851aa1d642e872df02d282fbcb29ee1b8e1bf150ffebf9bd4ce8c3670d43a284ff2c8197477a8cf6f71dfcd49b265ad938c723292ec57579998

data/.vimproject

@@ -8,11 +8,18 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
   alias
   find
   glob
+  doc
+  update
+  template
+  offsite
   workflow=workflow{
    task
    list
    info
-   task_old
+   install
+  }
+  resource=resource{
+   produce
   }
  }
  lib=lib {
@@ -99,24 +106,30 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
    workflow.rb
    workflow=workflow{
     definition.rb
+    documentation.rb
+    usage.rb
+    util.rb
     step.rb
     step=step{
-     info.rb
-     status.rb
-     load.rb
-     file.rb
      config.rb
      dependencies.rb
-     provenance.rb
+     file.rb
+     info.rb
+     load.rb
      progress.rb
+     provenance.rb
+     status.rb
+     inputs.rb
     }
     task.rb
     task=task{
+     dependencies.rb
      inputs.rb
     }
-    documentation.rb
-    usage.rb
-    util.rb
+    deployment.rb
+    deployment=deployment{
+     orchestrator.rb
+    }
    }
    semaphore.rb
    work_queue.rb
@@ -141,6 +154,8 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
     persist=persist{
      adapter.rb
      serialize.rb
+     sdbm.rb
+     tkrzw.rb
      tokyocabinet.rb
      fix_width_table.rb
     }
@@ -149,8 +164,18 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
     attach.rb
     change_id.rb
     path.rb
+    stream.rb
     open.rb
    }
+   offsite.rb
+   offsite=offsite{
+    ssh.rb
+    sync.rb
+    step.rb
+   }
+   hpc=hpc{
+    slurm.rb
+   }
   }
  }
  test=test {
@@ -181,7 +206,18 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
     test_task.rb
     test_usage.rb
     test_util.rb
+    step=step{
+     test_dependencies.rb
+     test_info.rb
+     test_load.rb
+     test_provenance.rb
+     test_status.rb
+    }
+    task=task{
+     test_dependencies.rb
+     test_inputs.rb
     }
+   }
    indiferent_hash=indiferent_hash{
     test_case_insensitive.rb
     test_options.rb
@@ -192,6 +228,9 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
   software=software{
    install_helpers
   }
+  templates=templates{
+   workflow.rb
+  }
  }
 
 

data/Rakefile

@@ -12,7 +12,7 @@ Juwelier::Tasks.new do |gem|
   gem.homepage = "http://github.com/mikisvaz/scout-gear"
   gem.license = "MIT"
   gem.summary = %Q{basic gear for scouts}
-  gem.description = %Q{Temporary files, logs, etc.}
+  gem.description = %Q{Temporary files, logs, path, resources, persistence, workflows, TSV, etc.}
   gem.email = "mikisvaz@gmail.com"
   gem.authors = ["Miguel Vazquez"]
 
@@ -21,6 +21,11 @@ Juwelier::Tasks.new do |gem|
   #  gem.add_runtime_dependency 'jabber4r', '> 0.1'
   #  gem.add_development_dependency 'rspec', '> 1.2.3'
   gem.add_runtime_dependency 'term-ansicolor'
+  gem.add_runtime_dependency 'net-ssh'
+  gem.add_runtime_dependency 'matrix'
+  gem.add_runtime_dependency 'sys-proctable'
+  gem.add_runtime_dependency 'RubyInline'
+  #gem.add_runtime_dependency 'tokyocabinet'
 
   gem.add_development_dependency "rdoc", "~> 3.12"
   gem.add_development_dependency "bundler", "~> 1.0"

data/VERSION

@@ -1 +1 @@
-8.0.0
+9.0.0

data/bin/scout

@@ -35,6 +35,12 @@ if dev_dir.nil?
   ARGV.delete _s if _s
 end
 
+if dev_dir.nil? && ENV["SCOUT_DEV"]
+  dev_dir = ENV["SCOUT_DEV"]
+  ARGV.delete "--dev"
+  ARGV.delete dev_dir
+end
+
 if dev_dir
   ['scout-*/lib'].each do |pattern|
     Dir.glob(File.join(File.expand_path(dev_dir), pattern)).each do |f|
@@ -48,6 +54,7 @@ if dev_dir
   end
 end
 
+require 'scout'
 
 require 'scout/simple_opt'
 
@@ -62,6 +69,7 @@ $ #{$0} <command> <subcommand> ... -a --arg1 --arg2='value' --arg3 'another-valu
 --nocolor               #{Log.color :yellow, "Disable colored output"}
 --nobar                 #{Log.color :yellow, "Disable progress report"}
 --locate_file           #{Log.color :yellow, "Report the location of the script instead of executing it"}
+-ck--config_keys*       #{Log.color :yellow, "Override some config keys"}
 EOF
 
 Log.nocolor = true if options[:nocolor]
@@ -84,6 +92,14 @@ else
   end
 end
 
+if config_keys = options.delete(:config_keys)
+  config_keys.split(",").each do |config|
+    config = config.strip
+    Scout::Config.process_config config
+  end
+end
+
+
 $scout_command_dir = Scout.bin.scout
 $scout_command_dir.path_maps[:scout_commands] = File.join(File.dirname(__dir__), "{PATH/bin\\/scout/scout_commands}")
 
@@ -217,7 +233,6 @@ begin
 
   scout_usage($previous_commands)
   exit_status = 0
-  exit exit_status
 
 rescue ParameterException
   puts
@@ -225,14 +240,11 @@ rescue ParameterException
   print_error($!.message, $!.backtrace)
   puts
   exit_status = -1
-  exit exit_status
 rescue SystemExit,CmdStop
   exit_status = $!.status
-  exit exit_status
 rescue Exception
   Log.exception $!
   exit_status = -1
-  exit exit_status
 ensure
   if options[:profile]
     result = RubyProf.stop

data/doc/lib/scout/path.md

@@ -0,0 +1,35 @@
+Path
+===
+
+```ruby
+:current => '{PWD}/{TOPLEVEL}/{SUBPATH}',
+:user    => '{HOME}/.{PKGDIR}/{TOPLEVEL}/{SUBPATH}',
+:global  => '/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
+:usr     => '/usr/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
+:local   => '/usr/local/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
+:fast    => '/fast/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
+:cache   => '/cache/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
+:bulk    => '/bulk/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
+:lib     => '{LIBDIR}/{TOPLEVEL}/{SUBPATH}',
+:scout_gear => File.join(Path.caller_lib_dir(__FILE__), "{TOPLEVEL}/{SUBPATH}"),
+:tmp     => '/tmp/{PKGDIR}/{TOPLEVEL}/{SUBPATH}',
+:default => :user
+
+@@basic_map_order ||= %w(current workflow user local global lib fast cache bulk)
+
+```
+
+
+
+# tags
+{PKGDIR}
+{LIBDIR}
+{RESOURCE}
+{HOME}
+{PWD}
+{TOPLEVEL}
+{SUBPATH}
+{BASENAME}
+{PATH}
+{MAPNAME}
+{REMOVE}

data/doc/lib/scout/workflow/task.md

@@ -0,0 +1,13 @@
+Task
+====
+
+```ruby
+    wf = Workflow.annonymous_workflow "TaskInputs" do
+      input :input1, :integer
+      task :step1 => :integer do |i| i end
+
+      dep :step1
+      input :input2, :integer, "Integer", 3
+      task :step2 => :integer do |i| i * step(:step1).load end
+    end
+```

data/lib/rbbt-scout.rb

@@ -1,6 +1,7 @@
 $LOAD_PATH.unshift File.join(__dir__, '../modules/rbbt-util/lib')
 module Rbbt
   extend Resource
-  self.pkgdir = 'rbbt'
   self.path_maps = Path.path_maps.merge(:rbbt_lib => File.expand_path(File.join(__dir__, '../modules/rbbt-util/', '{TOPLEVEL}','{SUBPATH}')))
 end
+
+Resource.set_software_env Rbbt.software

data/lib/scout/cmd.rb

@@ -14,23 +14,28 @@ module CMD
 
   def self.conda(tool, env = nil, channel = 'bioconda')
     if env
-      CMD.cmd("bash -l -c '(conda activate #{env} && conda install #{tool} -c #{channel})'")      
+      CMD.cmd("bash -l -c '(conda activate #{env} && conda install #{tool} -c #{channel})'")
     else
-      CMD.cmd("bash -l -c 'conda install #{tool} -c #{channel}'")      
+      CMD.cmd("bash -l -c 'conda install #{tool} -c #{channel}'")
     end
   end
 
+
   def self.get_tool(tool)
     return tool.to_s unless TOOLS[tool]
 
     @@init_cmd_tool ||= IndiferentHash.setup({})
+
+    claim, test, block, cmd = TOOLS[tool]
+    cmd = tool.to_s if cmd.nil?
+
     if !@@init_cmd_tool[tool]
-      claim, test, block, cmd = TOOLS[tool]
+
       begin
         if test
           CMD.cmd(test + " ")
         else
-          CMD.cmd("#{tool} --help")
+          CMD.cmd("#{cmd} --help")
         end
       rescue
         if claim
@@ -43,7 +48,7 @@ module CMD
       version = nil
       ["--version", "-version", "--help", ""].each do |f|
         begin
-          version_txt += CMD.cmd("#{tool} #{f} 2>&1", :nofail => true).read
+          version_txt += CMD.cmd("#{cmd} #{f} 2>&1", :nofail => true).read
           version = CMD.scan_version_text(version_txt, tool)
           break if version
         rescue
@@ -56,11 +61,11 @@ module CMD
       return cmd if cmd
     end
 
-    tool.to_s
+    cmd
   end
-
   def self.scan_version_text(text, cmd = nil)
     cmd = "NOCMDGIVE" if cmd.nil? || cmd.empty?
+    text = Misc.fixutf8 text
     text.split("\n").each do |line|
       next unless line =~ /\W#{cmd}\W/i
       m = line.match(/(v(?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
@@ -98,8 +103,8 @@ module CMD
 
       option = "--" << option.to_s if add_dashes and option.to_s[0] != '-'
 
-      case 
-      when value.nil? || FalseClass === value 
+      case
+      when value.nil? || FalseClass === value
         next
       when TrueClass === value
         string << "#{option} "
@@ -136,8 +141,8 @@ module CMD
     dont_close_in  = options.delete(:dont_close_in)
 
     log = true if log.nil?
-    
-    if cmd.nil? && ! Symbol === tool 
+
+    if cmd.nil? && ! Symbol === tool
       cmd = tool
     else
       tool = get_tool(tool)
@@ -163,7 +168,7 @@ module CMD
 
     cmd_options = process_cmd_options options
     if cmd =~ /'\{opt\}'/
-      cmd.sub!('\'{opt}\'', cmd_options) 
+      cmd.sub!('\'{opt}\'', cmd_options)
     else
       cmd << " " << cmd_options
     end
@@ -179,29 +184,28 @@ module CMD
                                 end
     pid = wait_thr.pid
 
-    Log.debug{"CMD: [#{pid}] #{cmd}" if log}
+    Log.debug{"CMD: [#{pid}] #{cmd}".strip if log}
 
     if in_content.respond_to?(:read)
       in_thread = Thread.new(Thread.current) do |parent|
-        Thread.current.report_on_exception = false if no_fail
         begin
-          begin
-            while c = in_content.readpartial(Open::BLOCK_SIZE)
-              sin << c 
-            end
-          rescue EOFError
+          Thread.current.report_on_exception = false if no_fail
+          Thread.current["name"] = "CMD in"
+          while c = in_content.read(Open::BLOCK_SIZE)
+            sin << c
           end
           sin.close  unless sin.closed?
 
           unless dont_close_in
-            in_content.close unless in_content.closed? 
-            in_content.join if in_content.respond_to? :join 
+            in_content.close unless in_content.closed?
+            in_content.join if in_content.respond_to? :join
           end
         rescue
           Log.error "Error in CMD  [#{pid}] #{cmd}: #{$!.message}" unless no_fail
           raise $!
         end
       end
+      Thread.pass until in_thread["name"]
     else
       in_thread = nil
       sin.close
@@ -211,7 +215,7 @@ module CMD
 
     if pipe
 
-      ConcurrentStream.setup sout, :pids => pids, :autojoin => autojoin, :no_fail => no_fail 
+      ConcurrentStream.setup sout, :pids => pids, :autojoin => autojoin, :no_fail => no_fail
 
       sout.callback = post if post
 
@@ -224,7 +228,7 @@ module CMD
             sout.log = line
             sout.std_err << line if save_stderr
             Log.log "STDERR [#{pid}]: " +  line, stderr if log
-          end 
+          end
           serr.close
           rescue
             Log.exception $!
@@ -244,8 +248,8 @@ module CMD
         err = ""
         err_thread = Thread.new do
           while not serr.eof?
-            line = serr.gets 
-            bar.process(line) 
+            line = serr.gets
+            bar.process(line)
             err << line if Integer === stderr and log
           end
           serr.close
@@ -254,18 +258,17 @@ module CMD
         err = ""
         err_thread = Thread.new do
           while not serr.eof?
-            err << serr.gets 
+            err << serr.gets
           end
           serr.close
         end
       else
         Open.consume_stream(serr, true)
-        #serr.close 
         err_thread = nil
         err = ""
       end
 
-      ConcurrentStream.setup sout, :pids => pids, :threads => [in_thread, err_thread].compact, :autojoin => autojoin, :no_fail => no_fail 
+      ConcurrentStream.setup sout, :pids => pids, :threads => [in_thread, err_thread].compact, :autojoin => autojoin, :no_fail => no_fail
 
       begin
         out = StringIO.new sout.read
@@ -324,7 +327,7 @@ module CMD
         starting = true
         line = "" if bar
       end
-    end 
+    end
     begin
       io.join
       bar.remove if bar

data/lib/scout/concurrent_stream.rb

@@ -12,13 +12,12 @@ module ConcurrentStream
   attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined, :aborted, :autojoin, :lock, :no_fail, :pair, :thread, :stream_exception, :log, :std_err, :next
 
   def self.setup(stream, options = {}, &block)
-    
     threads, pids, callback, abort_callback, filename, autojoin, lock, no_fail, pair, next_stream = IndiferentHash.process_options options, :threads, :pids, :callback, :abort_callback, :filename, :autojoin, :lock, :no_fail, :pair, :next
     stream.extend ConcurrentStream unless ConcurrentStream === stream
 
     stream.threads ||= []
     stream.pids ||= []
-    stream.threads.concat(Array === threads ? threads : [threads]) unless threads.nil? 
+    stream.threads.concat(Array === threads ? threads : [threads]) unless threads.nil?
     stream.pids.concat(Array === pids ? pids : [pids]) unless pids.nil? or pids.empty?
     stream.autojoin = autojoin unless autojoin.nil?
     stream.no_fail = no_fail unless no_fail.nil?
@@ -36,7 +35,7 @@ module ConcurrentStream
           callback.call
         end
       else
-        stream.callback = callback 
+        stream.callback = callback
       end
     end
 
@@ -48,7 +47,7 @@ module ConcurrentStream
           abort_callback.call
         end
       else
-        stream.abort_callback = abort_callback 
+        stream.abort_callback = abort_callback
       end
     end
 
@@ -80,7 +79,7 @@ module ConcurrentStream
 
   def join_threads
     if @threads
-      @threads.each do |t| 
+      @threads.each do |t|
         next if t == Thread.current
         begin
           t.join
@@ -101,7 +100,7 @@ module ConcurrentStream
             Log.low "Not failing on exception joining thread in ConcurrenStream - #{filename} - #{$!.message}"
           else
             Log.low "Exception joining thread in ConcurrenStream #{Log.fingerprint self} - #{Log.fingerprint t} - #{$!.message}"
-            stream_raise_exception $! 
+            stream_raise_exception $!
           end
         end
       end
@@ -111,13 +110,13 @@ module ConcurrentStream
 
   def join_pids
     if @pids and @pids.any?
-      @pids.each do |pid| 
+      @pids.each do |pid|
         begin
           Process.waitpid(pid, Process::WUNTRACED)
           stream_raise_exception ConcurrentStreamProcessFailed.new(pid, "Error in waitpid", self) unless $?.success? or no_fail
         rescue Errno::ECHILD
         end
-      end 
+      end
       @pids = []
     end
   end
@@ -143,7 +142,7 @@ module ConcurrentStream
       @joined = true
       begin
         lock.unlock if lock && lock.locked?
-      rescue 
+      rescue
         Log.exception $!
       end
       raise stream_exception if stream_exception
@@ -158,7 +157,7 @@ module ConcurrentStream
 
     threads = @threads.dup
     @threads.clear
-    threads.each do |t| 
+    threads.each do |t|
       next if t == Thread.current
       next if t["aborted"]
       t["aborted"] = true
@@ -166,14 +165,14 @@ module ConcurrentStream
       Log.debug "Aborting thread #{Log.fingerprint(t)} with exception: #{exception}"
       t.raise(exception)
       t.join
-    end 
+    end
   end
 
   def abort_pids
     @pids.each do |pid|
-      begin 
+      begin
         Log.low "Killing PID #{pid} in ConcurrentStream #{filename}"
-        Process.kill :INT, pid 
+        Process.kill :INT, pid
       rescue Errno::ESRCH
       end
     end if @pids
@@ -189,7 +188,7 @@ module ConcurrentStream
       Log.medium "Aborting stream #{Log.fingerprint self} [#{@aborted}]"
     end
     AbortedStream.setup(self, exception)
-    @aborted = true 
+    @aborted = true
     begin
       @abort_callback.call exception if @abort_callback
 
@@ -207,7 +206,7 @@ module ConcurrentStream
       close unless closed?
 
       if lock and lock.locked?
-        lock.unlock 
+        lock.unlock
       end
     end
   end
@@ -218,7 +217,7 @@ module ConcurrentStream
         super(*args)
       rescue
         self.abort
-        self.join 
+        self.join
         stream_raise_exception $!
       ensure
         self.join if ! @stream_exception && (self.closed? || self.eof?)
@@ -249,19 +248,37 @@ module ConcurrentStream
 
   def add_callback(&block)
     old_callback = callback
-    @callback = Proc.new do 
+    @callback = Proc.new do
       old_callback.call if old_callback
       block.call
     end
   end
 
   def stream_raise_exception(exception)
+    self.stream_exception = exception
     threads.each do |thread|
       thread.raise exception
     end
-    self.stream_exception = exception
-
     self.abort
   end
 
+  def self.process_stream(stream, close: true, join: true, message: "process_stream", **kwargs, &block)
+    ConcurrentStream.setup(stream, **kwargs)
+    begin
+      begin
+        yield
+      ensure
+        stream.close if close && stream.respond_to?(:close) && ! (stream.respond_to?(:closed?) && stream.closed?)
+        stream.join if join && stream.respond_to?(:join) && ! stream.joined?
+      end
+    rescue Aborted
+      Log.low "Aborted #{message}: #{$!.message}"
+      stream.abort($!) if stream.respond_to?(:abort) && ! stream.aborted?
+      raise $!
+    rescue Exception
+      Log.low "Exception #{message}: #{$!.message}"
+      stream.abort($!) if stream.respond_to?(:abort) && ! stream.aborted?
+      raise $!
+    end
+  end
 end

data/lib/scout/exceptions.rb

@@ -139,3 +139,13 @@ class SemaphoreInterrupted < TryAgain; end
 #
 #
 class ResourceNotFound < ScoutException; end
+
+class SSHProcessFailed < StandardError
+  attr_accessor :host, :cmd
+  def initialize(host, cmd)
+    @host = host
+    @cmd = cmd
+    message = "SSH server #{host} failed cmd '#{cmd}'" 
+    super(message)
+  end
+end

data/lib/scout/indiferent_hash.rb

@@ -75,5 +75,22 @@ module IndiferentHash
     end
     clean
   end
+
+  def slice(*list)
+    ext_list = []
+    list.each do |e|
+      case e
+      when Symbol
+        ext_list << e
+        ext_list << e.to_s
+      when String
+        ext_list << e
+        ext_list << e.to_sym
+      else
+        ext_list << e
+      end
+    end
+    IndiferentHash.setup(super(*ext_list))
+  end
 end