scout-gear 8.0.0 → 8.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 48f6bb392f3391e2816e701887bfd98617b1a68bc533c8b6de3307f25c84906d
-  data.tar.gz: 476858f63deaa7fa9fc531c7872bdaefa1d6b3ed32cdacbe41f53d1d170fcc87
+  metadata.gz: 9ca509bfb5065aabcdae73545139db0f8b04748084821e002c0475a55be5cb70
+  data.tar.gz: 6f0333ca6dd0ceee0ea9290cf3c30f7164c2f0732b76ce739e29da19a0687242
 SHA512:
-  metadata.gz: c61501c8cc79b57b64cb8613efc8b00fb84908bf0d55b008c6a6a39b99ea6bffa53630a8e134c07a49cb7bc4145ec181df057b6ba1eebfd2d722b2aa2d2e63d0
-  data.tar.gz: 100356a04cd34eba627239668747023ee5be4455ebbee715df39dbe04e9c8dfcac7007ce310e4622cf0c7e16ee9159e6fc2d4eefcaa486c24519c7f99185facf
+  metadata.gz: 2cbd82d97ec0dad50e624cda26ae6a05aeba8b9e5ff07a5b4746c677b8e2ab1996378237095ba09a70f8ed8cc926d782c1677aa24d6f94784f560090d6845a2c
+  data.tar.gz: aa9529b3393016c846b578b736428b964519d2fb5c1507a6335eff651d0588a5d9191cc61cff52279c18902f2efce386415099bcd92f8736f6d67cef1d33b561

data/.vimproject

@@ -8,11 +8,14 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
   alias
   find
   glob
+  doc
+  update
+  offsite
   workflow=workflow{
    task
    list
    info
-   task_old
+   install
   }
  }
  lib=lib {
@@ -99,24 +102,29 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
    workflow.rb
    workflow=workflow{
     definition.rb
+    documentation.rb
+    usage.rb
+    util.rb
     step.rb
     step=step{
-     info.rb
-     status.rb
-     load.rb
-     file.rb
      config.rb
      dependencies.rb
-     provenance.rb
+     file.rb
+     info.rb
+     load.rb
      progress.rb
+     provenance.rb
+     status.rb
     }
     task.rb
     task=task{
+     dependencies.rb
      inputs.rb
     }
-    documentation.rb
-    usage.rb
-    util.rb
+    deployment.rb
+    deployment=deployment{
+     orchestrator.rb
+    }
    }
    semaphore.rb
    work_queue.rb
@@ -141,6 +149,8 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
     persist=persist{
      adapter.rb
      serialize.rb
+     sdbm.rb
+     tkrzw.rb
      tokyocabinet.rb
      fix_width_table.rb
     }
@@ -149,8 +159,15 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
     attach.rb
     change_id.rb
     path.rb
+    stream.rb
     open.rb
    }
+   offsite.rb
+   offsite=offsite{
+    ssh.rb
+    sync.rb
+    step.rb
+   }
   }
  }
  test=test {

data/Rakefile

@@ -12,7 +12,7 @@ Juwelier::Tasks.new do |gem|
   gem.homepage = "http://github.com/mikisvaz/scout-gear"
   gem.license = "MIT"
   gem.summary = %Q{basic gear for scouts}
-  gem.description = %Q{Temporary files, logs, etc.}
+  gem.description = %Q{Temporary files, logs, path, resources, persistence, workflows, TSV, etc.}
   gem.email = "mikisvaz@gmail.com"
   gem.authors = ["Miguel Vazquez"]
 
@@ -21,6 +21,11 @@ Juwelier::Tasks.new do |gem|
   #  gem.add_runtime_dependency 'jabber4r', '> 0.1'
   #  gem.add_development_dependency 'rspec', '> 1.2.3'
   gem.add_runtime_dependency 'term-ansicolor'
+  gem.add_runtime_dependency 'net-ssh'
+  gem.add_runtime_dependency 'matrix'
+  gem.add_runtime_dependency 'sys-proctable'
+  gem.add_runtime_dependency 'RubyInline'
+  #gem.add_runtime_dependency 'tokyocabinet'
 
   gem.add_development_dependency "rdoc", "~> 3.12"
   gem.add_development_dependency "bundler", "~> 1.0"

data/VERSION

@@ -1 +1 @@
-8.0.0
+8.1.0

data/bin/scout

@@ -35,6 +35,12 @@ if dev_dir.nil?
   ARGV.delete _s if _s
 end
 
+if dev_dir.nil? && ENV["SCOUT_DEV"]
+  dev_dir = ENV["SCOUT_DEV"]
+  ARGV.delete "--dev"
+  ARGV.delete dev_dir
+end
+
 if dev_dir
   ['scout-*/lib'].each do |pattern|
     Dir.glob(File.join(File.expand_path(dev_dir), pattern)).each do |f|
@@ -62,6 +68,7 @@ $ #{$0} <command> <subcommand> ... -a --arg1 --arg2='value' --arg3 'another-valu
 --nocolor               #{Log.color :yellow, "Disable colored output"}
 --nobar                 #{Log.color :yellow, "Disable progress report"}
 --locate_file           #{Log.color :yellow, "Report the location of the script instead of executing it"}
+-ck--config_keys*       #{Log.color :yellow, "Override some config keys"}
 EOF
 
 Log.nocolor = true if options[:nocolor]
@@ -84,6 +91,14 @@ else
   end
 end
 
+if config_keys = options.delete(:config_keys)
+  config_keys.split(",").each do |config|
+    config = config.strip
+    Scout::Config.process_config config
+  end
+end
+
+
 $scout_command_dir = Scout.bin.scout
 $scout_command_dir.path_maps[:scout_commands] = File.join(File.dirname(__dir__), "{PATH/bin\\/scout/scout_commands}")
 
@@ -217,7 +232,6 @@ begin
 
   scout_usage($previous_commands)
   exit_status = 0
-  exit exit_status
 
 rescue ParameterException
   puts
@@ -225,14 +239,11 @@ rescue ParameterException
   print_error($!.message, $!.backtrace)
   puts
   exit_status = -1
-  exit exit_status
 rescue SystemExit,CmdStop
   exit_status = $!.status
-  exit exit_status
 rescue Exception
   Log.exception $!
   exit_status = -1
-  exit exit_status
 ensure
   if options[:profile]
     result = RubyProf.stop

data/doc/lib/scout/path.md

@@ -0,0 +1,35 @@
+Path
+===
+
+```ruby
+:current => '{PWD}/{TOPLEVEL}/{SUBPATH}',
+:user    => '{HOME}/.{PKGDIR}/{TOPLEVEL}/{SUBPATH}',
+:global  => '/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
+:usr     => '/usr/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
+:local   => '/usr/local/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
+:fast    => '/fast/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
+:cache   => '/cache/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
+:bulk    => '/bulk/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
+:lib     => '{LIBDIR}/{TOPLEVEL}/{SUBPATH}',
+:scout_gear => File.join(Path.caller_lib_dir(__FILE__), "{TOPLEVEL}/{SUBPATH}"),
+:tmp     => '/tmp/{PKGDIR}/{TOPLEVEL}/{SUBPATH}',
+:default => :user
+
+@@basic_map_order ||= %w(current workflow user local global lib fast cache bulk)
+
+```
+
+
+
+# tags
+{PKGDIR}
+{LIBDIR}
+{RESOURCE}
+{HOME}
+{PWD}
+{TOPLEVEL}
+{SUBPATH}
+{BASENAME}
+{PATH}
+{MAPNAME}
+{REMOVE}

data/doc/lib/scout/workflow/task.md

@@ -0,0 +1,13 @@
+Task
+====
+
+```ruby
+    wf = Workflow.annonymous_workflow "TaskInputs" do
+      input :input1, :integer
+      task :step1 => :integer do |i| i end
+
+      dep :step1
+      input :input2, :integer, "Integer", 3
+      task :step2 => :integer do |i| i * step(:step1).load end
+    end
+```

data/lib/scout/cmd.rb

@@ -14,9 +14,9 @@ module CMD
 
   def self.conda(tool, env = nil, channel = 'bioconda')
     if env
-      CMD.cmd("bash -l -c '(conda activate #{env} && conda install #{tool} -c #{channel})'")      
+      CMD.cmd("bash -l -c '(conda activate #{env} && conda install #{tool} -c #{channel})'")
     else
-      CMD.cmd("bash -l -c 'conda install #{tool} -c #{channel}'")      
+      CMD.cmd("bash -l -c 'conda install #{tool} -c #{channel}'")
     end
   end
 
@@ -61,6 +61,7 @@ module CMD
 
   def self.scan_version_text(text, cmd = nil)
     cmd = "NOCMDGIVE" if cmd.nil? || cmd.empty?
+    text = Misc.fixutf8 text
     text.split("\n").each do |line|
       next unless line =~ /\W#{cmd}\W/i
       m = line.match(/(v(?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
@@ -98,8 +99,8 @@ module CMD
 
       option = "--" << option.to_s if add_dashes and option.to_s[0] != '-'
 
-      case 
-      when value.nil? || FalseClass === value 
+      case
+      when value.nil? || FalseClass === value
         next
       when TrueClass === value
         string << "#{option} "
@@ -136,8 +137,8 @@ module CMD
     dont_close_in  = options.delete(:dont_close_in)
 
     log = true if log.nil?
-    
-    if cmd.nil? && ! Symbol === tool 
+
+    if cmd.nil? && ! Symbol === tool
       cmd = tool
     else
       tool = get_tool(tool)
@@ -163,7 +164,7 @@ module CMD
 
     cmd_options = process_cmd_options options
     if cmd =~ /'\{opt\}'/
-      cmd.sub!('\'{opt}\'', cmd_options) 
+      cmd.sub!('\'{opt}\'', cmd_options)
     else
       cmd << " " << cmd_options
     end
@@ -179,29 +180,28 @@ module CMD
                                 end
     pid = wait_thr.pid
 
-    Log.debug{"CMD: [#{pid}] #{cmd}" if log}
+    Log.debug{"CMD: [#{pid}] #{cmd}".strip if log}
 
     if in_content.respond_to?(:read)
       in_thread = Thread.new(Thread.current) do |parent|
-        Thread.current.report_on_exception = false if no_fail
         begin
-          begin
-            while c = in_content.readpartial(Open::BLOCK_SIZE)
-              sin << c 
-            end
-          rescue EOFError
+          Thread.current.report_on_exception = false if no_fail
+          Thread.current["name"] = "CMD in"
+          while c = in_content.read(Open::BLOCK_SIZE)
+            sin << c
           end
           sin.close  unless sin.closed?
 
           unless dont_close_in
-            in_content.close unless in_content.closed? 
-            in_content.join if in_content.respond_to? :join 
+            in_content.close unless in_content.closed?
+            in_content.join if in_content.respond_to? :join
           end
         rescue
           Log.error "Error in CMD  [#{pid}] #{cmd}: #{$!.message}" unless no_fail
           raise $!
         end
       end
+      Thread.pass until in_thread["name"]
     else
       in_thread = nil
       sin.close
@@ -211,7 +211,7 @@ module CMD
 
     if pipe
 
-      ConcurrentStream.setup sout, :pids => pids, :autojoin => autojoin, :no_fail => no_fail 
+      ConcurrentStream.setup sout, :pids => pids, :autojoin => autojoin, :no_fail => no_fail
 
       sout.callback = post if post
 
@@ -224,7 +224,7 @@ module CMD
             sout.log = line
             sout.std_err << line if save_stderr
             Log.log "STDERR [#{pid}]: " +  line, stderr if log
-          end 
+          end
           serr.close
           rescue
             Log.exception $!
@@ -244,8 +244,8 @@ module CMD
         err = ""
         err_thread = Thread.new do
           while not serr.eof?
-            line = serr.gets 
-            bar.process(line) 
+            line = serr.gets
+            bar.process(line)
             err << line if Integer === stderr and log
           end
           serr.close
@@ -254,18 +254,17 @@ module CMD
         err = ""
         err_thread = Thread.new do
           while not serr.eof?
-            err << serr.gets 
+            err << serr.gets
           end
           serr.close
         end
       else
         Open.consume_stream(serr, true)
-        #serr.close 
         err_thread = nil
         err = ""
       end
 
-      ConcurrentStream.setup sout, :pids => pids, :threads => [in_thread, err_thread].compact, :autojoin => autojoin, :no_fail => no_fail 
+      ConcurrentStream.setup sout, :pids => pids, :threads => [in_thread, err_thread].compact, :autojoin => autojoin, :no_fail => no_fail
 
       begin
         out = StringIO.new sout.read
@@ -324,7 +323,7 @@ module CMD
         starting = true
         line = "" if bar
       end
-    end 
+    end
     begin
       io.join
       bar.remove if bar

data/lib/scout/concurrent_stream.rb

@@ -12,13 +12,12 @@ module ConcurrentStream
   attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined, :aborted, :autojoin, :lock, :no_fail, :pair, :thread, :stream_exception, :log, :std_err, :next
 
   def self.setup(stream, options = {}, &block)
-    
     threads, pids, callback, abort_callback, filename, autojoin, lock, no_fail, pair, next_stream = IndiferentHash.process_options options, :threads, :pids, :callback, :abort_callback, :filename, :autojoin, :lock, :no_fail, :pair, :next
     stream.extend ConcurrentStream unless ConcurrentStream === stream
 
     stream.threads ||= []
     stream.pids ||= []
-    stream.threads.concat(Array === threads ? threads : [threads]) unless threads.nil? 
+    stream.threads.concat(Array === threads ? threads : [threads]) unless threads.nil?
     stream.pids.concat(Array === pids ? pids : [pids]) unless pids.nil? or pids.empty?
     stream.autojoin = autojoin unless autojoin.nil?
     stream.no_fail = no_fail unless no_fail.nil?
@@ -36,7 +35,7 @@ module ConcurrentStream
           callback.call
         end
       else
-        stream.callback = callback 
+        stream.callback = callback
       end
     end
 
@@ -48,7 +47,7 @@ module ConcurrentStream
           abort_callback.call
         end
       else
-        stream.abort_callback = abort_callback 
+        stream.abort_callback = abort_callback
       end
     end
 
@@ -80,7 +79,7 @@ module ConcurrentStream
 
   def join_threads
     if @threads
-      @threads.each do |t| 
+      @threads.each do |t|
         next if t == Thread.current
         begin
           t.join
@@ -101,7 +100,7 @@ module ConcurrentStream
             Log.low "Not failing on exception joining thread in ConcurrenStream - #{filename} - #{$!.message}"
           else
             Log.low "Exception joining thread in ConcurrenStream #{Log.fingerprint self} - #{Log.fingerprint t} - #{$!.message}"
-            stream_raise_exception $! 
+            stream_raise_exception $!
           end
         end
       end
@@ -111,13 +110,13 @@ module ConcurrentStream
 
   def join_pids
     if @pids and @pids.any?
-      @pids.each do |pid| 
+      @pids.each do |pid|
         begin
           Process.waitpid(pid, Process::WUNTRACED)
           stream_raise_exception ConcurrentStreamProcessFailed.new(pid, "Error in waitpid", self) unless $?.success? or no_fail
         rescue Errno::ECHILD
         end
-      end 
+      end
       @pids = []
     end
   end
@@ -143,7 +142,7 @@ module ConcurrentStream
       @joined = true
       begin
         lock.unlock if lock && lock.locked?
-      rescue 
+      rescue
         Log.exception $!
       end
       raise stream_exception if stream_exception
@@ -158,7 +157,7 @@ module ConcurrentStream
 
     threads = @threads.dup
     @threads.clear
-    threads.each do |t| 
+    threads.each do |t|
       next if t == Thread.current
       next if t["aborted"]
       t["aborted"] = true
@@ -166,14 +165,14 @@ module ConcurrentStream
       Log.debug "Aborting thread #{Log.fingerprint(t)} with exception: #{exception}"
       t.raise(exception)
       t.join
-    end 
+    end
   end
 
   def abort_pids
     @pids.each do |pid|
-      begin 
+      begin
         Log.low "Killing PID #{pid} in ConcurrentStream #{filename}"
-        Process.kill :INT, pid 
+        Process.kill :INT, pid
       rescue Errno::ESRCH
       end
     end if @pids
@@ -189,7 +188,7 @@ module ConcurrentStream
       Log.medium "Aborting stream #{Log.fingerprint self} [#{@aborted}]"
     end
     AbortedStream.setup(self, exception)
-    @aborted = true 
+    @aborted = true
     begin
       @abort_callback.call exception if @abort_callback
 
@@ -207,7 +206,7 @@ module ConcurrentStream
       close unless closed?
 
       if lock and lock.locked?
-        lock.unlock 
+        lock.unlock
       end
     end
   end
@@ -218,7 +217,7 @@ module ConcurrentStream
         super(*args)
       rescue
         self.abort
-        self.join 
+        self.join
         stream_raise_exception $!
       ensure
         self.join if ! @stream_exception && (self.closed? || self.eof?)
@@ -249,19 +248,37 @@ module ConcurrentStream
 
   def add_callback(&block)
     old_callback = callback
-    @callback = Proc.new do 
+    @callback = Proc.new do
       old_callback.call if old_callback
       block.call
     end
   end
 
   def stream_raise_exception(exception)
+    self.stream_exception = exception
     threads.each do |thread|
       thread.raise exception
     end
-    self.stream_exception = exception
-
     self.abort
   end
 
+  def self.process_stream(stream, close: true, join: true, message: "process_stream", **kwargs, &block)
+    ConcurrentStream.setup(stream, **kwargs)
+    begin
+      begin
+        yield
+      ensure
+        stream.close if close && stream.respond_to?(:close) && ! (stream.respond_to?(:closed?) && stream.closed?)
+        stream.join if join && stream.respond_to?(:join) && ! stream.joined?
+      end
+    rescue Aborted
+      Log.low "Aborted #{message}: #{$!.message}"
+      stream.abort($!) if stream.respond_to?(:abort) && ! stream.aborted?
+      raise $!
+    rescue Exception
+      Log.low "Exception #{message}: #{$!.message}"
+      stream.abort($!) if stream.respond_to?(:abort) && ! stream.aborted?
+      raise $!
+    end
+  end
 end

data/lib/scout/exceptions.rb

@@ -139,3 +139,13 @@ class SemaphoreInterrupted < TryAgain; end
 #
 #
 class ResourceNotFound < ScoutException; end
+
+class SSHProcessFailed < StandardError
+  attr_accessor :host, :cmd
+  def initialize(host, cmd)
+    @host = host
+    @cmd = cmd
+    message = "SSH server #{host} failed cmd '#{cmd}'" 
+    super(message)
+  end
+end

data/lib/scout/log/color.rb

@@ -10,8 +10,8 @@ module Colorize
 
   def self.colors
     @colors ||= IndiferentHash.setup(Hash[<<-EOF.split("\n").collect{|l| l.split(" ")}])
-green #00cd00 
-red #cd0000 
+green #00cd00
+red #cd0000
 yellow #ffd700
 blue #0000cd
 path blue
@@ -49,26 +49,26 @@ EOF
     when "black"
       '#fff'
     when 'green'
-      colors["green3"] 
+      colors["green3"]
     when 'red'
-      colors["red3"] 
+      colors["red3"]
     when 'yellow'
-      colors["gold1"] 
+      colors["gold1"]
     when 'blue'
-      colors["RoyalBlue"] 
+      colors["RoyalBlue"]
     else
       colors[color.to_s] || color.to_s
     end
   end
 
-  def self.continuous(array, start = "#40324F", eend = "#EABD5D", percent = false) 
+  def self.continuous(array, start = "#40324F", eend = "#EABD5D", percent = false)
     start_color = Color.new from_name(start)
     end_color = Color.new from_name(eend)
 
     if percent
       array = array.collect{|v| n = v.to_f; n = n > 100 ? 100 : n; n < 0.001 ? 0.001 : n}
     else
-      array = array.collect{|v| v.to_f } 
+      array = array.collect{|v| v.to_f }
     end
     max = array.max
     min = array.min
@@ -157,7 +157,7 @@ module Log
     :workflow => yellow,
   })
   HIGHLIGHT = "\033[1m"
-  
+
   def self.uncolor(str)
     "" << Term::ANSIColor.uncolor(str)
   end
@@ -167,7 +167,7 @@ module Log
   end
 
   def self.color(color, str = nil, reset = false)
-    return str.dup || "" if nocolor 
+    return str.dup || "" if nocolor
 
     if (color == :integer || color == :float) && Numeric === str
       color = if str < 0
@@ -179,7 +179,7 @@ module Log
               end
     end
 
-    if color == :status 
+    if color == :status
       color = case str.to_sym
               when :done
                 :green

data/lib/scout/log/progress/report.rb

@@ -4,8 +4,7 @@ module Log
     def print(io, str)
       return if self.severity && self.severity < Log.severity
       return if Log.no_bar
-      STDERR.print str
-      Log.logfile.puts str unless Log.logfile.nil?
+      Log.log_write str
       Log::LAST.replace "progress"
     end
 
@@ -57,9 +56,12 @@ module Log
         thr = short_mean
       else
         thr = begin
-                (@ticks || 1) / (Time.now - @start) 
-              rescue
-                1
+                d = Time.now - @start
+                if d == 0
+                  1
+                else
+                  (@ticks || 1) / d
+                end
               end
       end
 

data/lib/scout/log/progress/util.rb

@@ -74,6 +74,7 @@ module Log
       BAR_MUTEX.synchronize do
         REMOVE << bar
       end
+      cleanup_bars
       Log::LAST.replace "remove_bar" if Log::LAST == "progress"
     end
 
@@ -128,6 +129,8 @@ module Log
             nil
           end
         end
+      rescue Interrupt
+        raise $!
       rescue Exception
         nil
       end

data/lib/scout/log/trap.rb

@@ -1,7 +1,7 @@
 module Log
   def self.trap_std(msg = "STDOUT", msge = "STDERR", severity = 0, severity_err = nil)
-    sout, sin = Misc.pipe
-    soute, sine = Misc.pipe
+    sout, sin = Open.pipe
+    soute, sine = Open.pipe
     backup_stderr = STDERR.dup
     backup_stdout = STDOUT.dup
     old_logfile = Log.logfile
@@ -38,7 +38,7 @@ module Log
   end
 
   def self.trap_stderr(msg = "STDERR", severity = 0)
-    sout, sin = Misc.pipe
+    sout, sin = Open.pipe
     backup_stderr = STDERR.dup
     old_logfile = Log.logfile
     Log.logfile(backup_stderr)