scout-gear 7.3.0 → 8.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7273030ac34bd180620d9075ee85b26d8a9883831a5925fc911915f1edeeecf3
-  data.tar.gz: 4e445acb896844240c0024b7754cffee93af71feb62f9c4251fda70aee79121a
+  metadata.gz: 9ca509bfb5065aabcdae73545139db0f8b04748084821e002c0475a55be5cb70
+  data.tar.gz: 6f0333ca6dd0ceee0ea9290cf3c30f7164c2f0732b76ce739e29da19a0687242
 SHA512:
-  metadata.gz: '029a8eb23eda77c7e0c11ac2966e1e8ebf3eae479bbee0c76e113f5b839c655e97e0fec5025cc3c251795afcfc92ac3395a79b68c8eb99158d1cee8e2310f002'
-  data.tar.gz: 04ed4bcabfa7c41b80ba520074404bfb09c723ecdb8d02a2052f73fe86759bab7fdadff31f0f66284ed4938ebe5a1cad56fb94182b3edcc0de6325decdcb84f6
+  metadata.gz: 2cbd82d97ec0dad50e624cda26ae6a05aeba8b9e5ff07a5b4746c677b8e2ab1996378237095ba09a70f8ed8cc926d782c1677aa24d6f94784f560090d6845a2c
+  data.tar.gz: aa9529b3393016c846b578b736428b964519d2fb5c1507a6335eff651d0588a5d9191cc61cff52279c18902f2efce386415099bcd92f8736f6d67cef1d33b561

data/.vimproject

@@ -8,11 +8,14 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
   alias
   find
   glob
+  doc
+  update
+  offsite
   workflow=workflow{
-   task_old
    task
    list
    info
+   install
   }
  }
  lib=lib {
@@ -32,7 +35,6 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
     filesystem.rb
     monitor.rb
     system.rb
-    helper.rb
    }
    named_array.rb
    indiferent_hash.rb
@@ -50,6 +52,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
      report.rb
      util.rb
     }
+    trap.rb
    }
    tmpfile.rb
    simple_opt.rb
@@ -71,6 +74,9 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
    open.rb
    open=open{
     lock.rb
+    lock=lock{
+     lockfile.rb
+    }
     stream.rb
     remote.rb
     util.rb
@@ -96,23 +102,29 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
    workflow.rb
    workflow=workflow{
     definition.rb
+    documentation.rb
+    usage.rb
+    util.rb
     step.rb
     step=step{
-     info.rb
-     load.rb
-     file.rb
      config.rb
      dependencies.rb
-     provenance.rb
+     file.rb
+     info.rb
+     load.rb
      progress.rb
+     provenance.rb
+     status.rb
     }
     task.rb
     task=task{
+     dependencies.rb
      inputs.rb
     }
-    documentation.rb
-    usage.rb
-    util.rb
+    deployment.rb
+    deployment=deployment{
+     orchestrator.rb
+    }
    }
    semaphore.rb
    work_queue.rb
@@ -122,26 +134,40 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
    }
    tsv.rb
    tsv=tsv{
+    util.rb
+    util=util{
+     process.rb
+     select.rb
+     process.rb
+     unzip.rb
+     reorder.rb
+    }
     parser.rb
     dumper.rb
+    transformer.rb
     persist.rb
     persist=persist{
      adapter.rb
      serialize.rb
+     sdbm.rb
+     tkrzw.rb
      tokyocabinet.rb
      fix_width_table.rb
     }
     index.rb
     traverse.rb
-    util.rb
-    util=util{
-     process.rb
-     select.rb
-     filter.rb
-    }
+    attach.rb
+    change_id.rb
     path.rb
+    stream.rb
     open.rb
    }
+   offsite.rb
+   offsite=offsite{
+    ssh.rb
+    sync.rb
+    step.rb
+   }
   }
  }
  test=test {
@@ -309,6 +335,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
 
      workflow.rb
      workflow=workflow{
+      refactor.rb
       annotate.rb
 
       definition.rb
@@ -390,6 +417,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
 
      tsv.rb
      tsv=tsv{
+      refactor.rb
       parser.rb
       dumper.rb
 
@@ -513,7 +541,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
       test_path.rb
      }
      workflow=workflow{
-      _test_soap.rb
+      test_soap.rb
       test_doc.rb
       test_step.rb
       test_task.rb

data/Rakefile

@@ -12,7 +12,7 @@ Juwelier::Tasks.new do |gem|
   gem.homepage = "http://github.com/mikisvaz/scout-gear"
   gem.license = "MIT"
   gem.summary = %Q{basic gear for scouts}
-  gem.description = %Q{Temporary files, logs, etc.}
+  gem.description = %Q{Temporary files, logs, path, resources, persistence, workflows, TSV, etc.}
   gem.email = "mikisvaz@gmail.com"
   gem.authors = ["Miguel Vazquez"]
 
@@ -21,6 +21,11 @@ Juwelier::Tasks.new do |gem|
   #  gem.add_runtime_dependency 'jabber4r', '> 0.1'
   #  gem.add_development_dependency 'rspec', '> 1.2.3'
   gem.add_runtime_dependency 'term-ansicolor'
+  gem.add_runtime_dependency 'net-ssh'
+  gem.add_runtime_dependency 'matrix'
+  gem.add_runtime_dependency 'sys-proctable'
+  gem.add_runtime_dependency 'RubyInline'
+  #gem.add_runtime_dependency 'tokyocabinet'
 
   gem.add_development_dependency "rdoc", "~> 3.12"
   gem.add_development_dependency "bundler", "~> 1.0"

data/VERSION

@@ -1 +1 @@
-7.3.0
+8.1.0

data/bin/scout

@@ -2,6 +2,10 @@
 
 $LOAD_PATH.unshift File.join(__dir__, '../lib')
 
+ENV["SCOUT_NOCOLOR"] = "true" if ARGV.include? "--nocolor"
+
+ENV["SCOUT_NO_PROGRESS"] = "true" if ARGV.include? "--nobar"
+
 require 'scout-gear'
 
 class CmdStop < Exception
@@ -31,6 +35,12 @@ if dev_dir.nil?
   ARGV.delete _s if _s
 end
 
+if dev_dir.nil? && ENV["SCOUT_DEV"]
+  dev_dir = ENV["SCOUT_DEV"]
+  ARGV.delete "--dev"
+  ARGV.delete dev_dir
+end
+
 if dev_dir
   ['scout-*/lib'].each do |pattern|
     Dir.glob(File.join(File.expand_path(dev_dir), pattern)).each do |f|
@@ -44,7 +54,6 @@ if dev_dir
   end
 end
 
-Log.nocolor = true if ARGV.include? "--nocolor"
 
 require 'scout/simple_opt'
 
@@ -59,6 +68,7 @@ $ #{$0} <command> <subcommand> ... -a --arg1 --arg2='value' --arg3 'another-valu
 --nocolor               #{Log.color :yellow, "Disable colored output"}
 --nobar                 #{Log.color :yellow, "Disable progress report"}
 --locate_file           #{Log.color :yellow, "Report the location of the script instead of executing it"}
+-ck--config_keys*       #{Log.color :yellow, "Override some config keys"}
 EOF
 
 Log.nocolor = true if options[:nocolor]
@@ -66,11 +76,11 @@ Log.nocolor = true if options[:nocolor]
 locate = options.delete :locate_file
 
 if options[:log_file]
-  Log.logfile(options[:log_file])
+  Log.logfile(options.delete(:log_file))
 end
 
 if options[:log]
-  Log.severity = options[:log].to_i
+  Log.severity = options.delete(:log).to_i
 else
   global_severity = Log.get_level(Scout.etc.log_severity.read.strip) if Scout.etc.log_severity.exists?
   if ENV["SCOUT_LOG"]
@@ -81,6 +91,14 @@ else
   end
 end
 
+if config_keys = options.delete(:config_keys)
+  config_keys.split(",").each do |config|
+    config = config.strip
+    Scout::Config.process_config config
+  end
+end
+
+
 $scout_command_dir = Scout.bin.scout
 $scout_command_dir.path_maps[:scout_commands] = File.join(File.dirname(__dir__), "{PATH/bin\\/scout/scout_commands}")
 
@@ -214,7 +232,6 @@ begin
 
   scout_usage($previous_commands)
   exit_status = 0
-  exit exit_status
 
 rescue ParameterException
   puts
@@ -222,14 +239,11 @@ rescue ParameterException
   print_error($!.message, $!.backtrace)
   puts
   exit_status = -1
-  exit exit_status
 rescue SystemExit,CmdStop
   exit_status = $!.status
-  exit exit_status
 rescue Exception
   Log.exception $!
   exit_status = -1
-  exit exit_status
 ensure
   if options[:profile]
     result = RubyProf.stop

data/doc/lib/scout/path.md

@@ -0,0 +1,35 @@
+Path
+===
+
+```ruby
+:current => '{PWD}/{TOPLEVEL}/{SUBPATH}',
+:user    => '{HOME}/.{PKGDIR}/{TOPLEVEL}/{SUBPATH}',
+:global  => '/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
+:usr     => '/usr/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
+:local   => '/usr/local/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
+:fast    => '/fast/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
+:cache   => '/cache/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
+:bulk    => '/bulk/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
+:lib     => '{LIBDIR}/{TOPLEVEL}/{SUBPATH}',
+:scout_gear => File.join(Path.caller_lib_dir(__FILE__), "{TOPLEVEL}/{SUBPATH}"),
+:tmp     => '/tmp/{PKGDIR}/{TOPLEVEL}/{SUBPATH}',
+:default => :user
+
+@@basic_map_order ||= %w(current workflow user local global lib fast cache bulk)
+
+```
+
+
+
+# tags
+{PKGDIR}
+{LIBDIR}
+{RESOURCE}
+{HOME}
+{PWD}
+{TOPLEVEL}
+{SUBPATH}
+{BASENAME}
+{PATH}
+{MAPNAME}
+{REMOVE}

data/doc/lib/scout/workflow/task.md

@@ -0,0 +1,13 @@
+Task
+====
+
+```ruby
+    wf = Workflow.annonymous_workflow "TaskInputs" do
+      input :input1, :integer
+      task :step1 => :integer do |i| i end
+
+      dep :step1
+      input :input2, :integer, "Integer", 3
+      task :step2 => :integer do |i| i * step(:step1).load end
+    end
+```

data/lib/rbbt-scout.rb

@@ -2,4 +2,5 @@ $LOAD_PATH.unshift File.join(__dir__, '../modules/rbbt-util/lib')
 module Rbbt
   extend Resource
   self.pkgdir = 'rbbt'
+  self.path_maps = Path.path_maps.merge(:rbbt_lib => File.expand_path(File.join(__dir__, '../modules/rbbt-util/', '{TOPLEVEL}','{SUBPATH}')))
 end

data/lib/scout/cmd.rb

@@ -14,9 +14,9 @@ module CMD
 
   def self.conda(tool, env = nil, channel = 'bioconda')
     if env
-      CMD.cmd("bash -l -c '(conda activate #{env} && conda install #{tool} -c #{channel})'")      
+      CMD.cmd("bash -l -c '(conda activate #{env} && conda install #{tool} -c #{channel})'")
     else
-      CMD.cmd("bash -l -c 'conda install #{tool} -c #{channel}'")      
+      CMD.cmd("bash -l -c 'conda install #{tool} -c #{channel}'")
     end
   end
 
@@ -61,6 +61,7 @@ module CMD
 
   def self.scan_version_text(text, cmd = nil)
     cmd = "NOCMDGIVE" if cmd.nil? || cmd.empty?
+    text = Misc.fixutf8 text
     text.split("\n").each do |line|
       next unless line =~ /\W#{cmd}\W/i
       m = line.match(/(v(?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
@@ -98,8 +99,8 @@ module CMD
 
       option = "--" << option.to_s if add_dashes and option.to_s[0] != '-'
 
-      case 
-      when value.nil? || FalseClass === value 
+      case
+      when value.nil? || FalseClass === value
         next
       when TrueClass === value
         string << "#{option} "
@@ -136,8 +137,8 @@ module CMD
     dont_close_in  = options.delete(:dont_close_in)
 
     log = true if log.nil?
-    
-    if cmd.nil? && ! Symbol === tool 
+
+    if cmd.nil? && ! Symbol === tool
       cmd = tool
     else
       tool = get_tool(tool)
@@ -163,7 +164,7 @@ module CMD
 
     cmd_options = process_cmd_options options
     if cmd =~ /'\{opt\}'/
-      cmd.sub!('\'{opt}\'', cmd_options) 
+      cmd.sub!('\'{opt}\'', cmd_options)
     else
       cmd << " " << cmd_options
     end
@@ -179,29 +180,28 @@ module CMD
                                 end
     pid = wait_thr.pid
 
-    Log.debug{"CMD: [#{pid}] #{cmd}" if log}
+    Log.debug{"CMD: [#{pid}] #{cmd}".strip if log}
 
     if in_content.respond_to?(:read)
       in_thread = Thread.new(Thread.current) do |parent|
-        Thread.current.report_on_exception = false if no_fail
         begin
-          begin
-            while c = in_content.readpartial(Open::BLOCK_SIZE)
-              sin << c 
-            end
-          rescue EOFError
+          Thread.current.report_on_exception = false if no_fail
+          Thread.current["name"] = "CMD in"
+          while c = in_content.read(Open::BLOCK_SIZE)
+            sin << c
           end
           sin.close  unless sin.closed?
 
           unless dont_close_in
-            in_content.close unless in_content.closed? 
-            in_content.join if in_content.respond_to? :join 
+            in_content.close unless in_content.closed?
+            in_content.join if in_content.respond_to? :join
           end
         rescue
           Log.error "Error in CMD  [#{pid}] #{cmd}: #{$!.message}" unless no_fail
           raise $!
         end
       end
+      Thread.pass until in_thread["name"]
     else
       in_thread = nil
       sin.close
@@ -211,7 +211,7 @@ module CMD
 
     if pipe
 
-      ConcurrentStream.setup sout, :pids => pids, :autojoin => autojoin, :no_fail => no_fail 
+      ConcurrentStream.setup sout, :pids => pids, :autojoin => autojoin, :no_fail => no_fail
 
       sout.callback = post if post
 
@@ -224,7 +224,7 @@ module CMD
             sout.log = line
             sout.std_err << line if save_stderr
             Log.log "STDERR [#{pid}]: " +  line, stderr if log
-          end 
+          end
           serr.close
           rescue
             Log.exception $!
@@ -244,8 +244,8 @@ module CMD
         err = ""
         err_thread = Thread.new do
           while not serr.eof?
-            line = serr.gets 
-            bar.process(line) 
+            line = serr.gets
+            bar.process(line)
             err << line if Integer === stderr and log
           end
           serr.close
@@ -254,25 +254,24 @@ module CMD
         err = ""
         err_thread = Thread.new do
           while not serr.eof?
-            err << serr.gets 
+            err << serr.gets
           end
           serr.close
         end
       else
         Open.consume_stream(serr, true)
-        #serr.close 
         err_thread = nil
         err = ""
       end
 
-      ConcurrentStream.setup sout, :pids => pids, :threads => [in_thread, err_thread].compact, :autojoin => autojoin, :no_fail => no_fail 
+      ConcurrentStream.setup sout, :pids => pids, :threads => [in_thread, err_thread].compact, :autojoin => autojoin, :no_fail => no_fail
 
       begin
         out = StringIO.new sout.read
         sout.close unless sout.closed?
 
         status = wait_thr.value
-        if not status.success? and not no_fail
+        if status && ! status.success? && ! no_fail
           if !err.empty?
             raise ProcessFailed.new pid, "#{cmd} failed with error status #{status.exitstatus}.\n#{err}"
           else
@@ -324,7 +323,7 @@ module CMD
         starting = true
         line = "" if bar
       end
-    end 
+    end
     begin
       io.join
       bar.remove if bar

data/lib/scout/concurrent_stream.rb

@@ -12,13 +12,12 @@ module ConcurrentStream
   attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined, :aborted, :autojoin, :lock, :no_fail, :pair, :thread, :stream_exception, :log, :std_err, :next
 
   def self.setup(stream, options = {}, &block)
-    
     threads, pids, callback, abort_callback, filename, autojoin, lock, no_fail, pair, next_stream = IndiferentHash.process_options options, :threads, :pids, :callback, :abort_callback, :filename, :autojoin, :lock, :no_fail, :pair, :next
     stream.extend ConcurrentStream unless ConcurrentStream === stream
 
     stream.threads ||= []
     stream.pids ||= []
-    stream.threads.concat(Array === threads ? threads : [threads]) unless threads.nil? 
+    stream.threads.concat(Array === threads ? threads : [threads]) unless threads.nil?
     stream.pids.concat(Array === pids ? pids : [pids]) unless pids.nil? or pids.empty?
     stream.autojoin = autojoin unless autojoin.nil?
     stream.no_fail = no_fail unless no_fail.nil?
@@ -36,7 +35,7 @@ module ConcurrentStream
           callback.call
         end
       else
-        stream.callback = callback 
+        stream.callback = callback
       end
     end
 
@@ -48,7 +47,7 @@ module ConcurrentStream
           abort_callback.call
         end
       else
-        stream.abort_callback = abort_callback 
+        stream.abort_callback = abort_callback
       end
     end
 
@@ -80,7 +79,7 @@ module ConcurrentStream
 
   def join_threads
     if @threads
-      @threads.each do |t| 
+      @threads.each do |t|
         next if t == Thread.current
         begin
           t.join
@@ -101,7 +100,7 @@ module ConcurrentStream
             Log.low "Not failing on exception joining thread in ConcurrenStream - #{filename} - #{$!.message}"
           else
             Log.low "Exception joining thread in ConcurrenStream #{Log.fingerprint self} - #{Log.fingerprint t} - #{$!.message}"
-            stream_raise_exception $! 
+            stream_raise_exception $!
           end
         end
       end
@@ -111,13 +110,13 @@ module ConcurrentStream
 
   def join_pids
     if @pids and @pids.any?
-      @pids.each do |pid| 
+      @pids.each do |pid|
         begin
           Process.waitpid(pid, Process::WUNTRACED)
           stream_raise_exception ConcurrentStreamProcessFailed.new(pid, "Error in waitpid", self) unless $?.success? or no_fail
         rescue Errno::ECHILD
         end
-      end 
+      end
       @pids = []
     end
   end
@@ -141,7 +140,11 @@ module ConcurrentStream
       close unless closed?
     ensure
       @joined = true
-      lock.unlock if lock and lock.locked?
+      begin
+        lock.unlock if lock && lock.locked?
+      rescue
+        Log.exception $!
+      end
       raise stream_exception if stream_exception
     end
   end
@@ -152,28 +155,24 @@ module ConcurrentStream
     name += " - file:#{filename}" if filename
     Log.low "Aborting threads (#{name}) - #{@threads.collect{|t| Log.fingerprint(t) } * ", "}"
 
-    @threads.each do |t| 
+    threads = @threads.dup
+    @threads.clear
+    threads.each do |t|
       next if t == Thread.current
+      next if t["aborted"]
+      t["aborted"] = true
+      exception = exception.nil? ? Aborted.new : exception
       Log.debug "Aborting thread #{Log.fingerprint(t)} with exception: #{exception}"
-      t.raise((exception.nil? ? Aborted.new : exception))
-    end 
-
-    @threads.each do |t|
-      next if t == Thread.current
-      begin
-        t.join
-      rescue Aborted
-      rescue Exception
-        Log.debug "Thread (#{name}) exception: #{$!.message}"
-      end
+      t.raise(exception)
+      t.join
     end
   end
 
   def abort_pids
     @pids.each do |pid|
-      begin 
+      begin
         Log.low "Killing PID #{pid} in ConcurrentStream #{filename}"
-        Process.kill :INT, pid 
+        Process.kill :INT, pid
       rescue Errno::ESRCH
       end
     end if @pids
@@ -189,7 +188,7 @@ module ConcurrentStream
       Log.medium "Aborting stream #{Log.fingerprint self} [#{@aborted}]"
     end
     AbortedStream.setup(self, exception)
-    @aborted = true 
+    @aborted = true
     begin
       @abort_callback.call exception if @abort_callback
 
@@ -207,7 +206,7 @@ module ConcurrentStream
       close unless closed?
 
       if lock and lock.locked?
-        lock.unlock 
+        lock.unlock
       end
     end
   end
@@ -217,12 +216,11 @@ module ConcurrentStream
       begin
         super(*args)
       rescue
-        Log.exception $!
         self.abort
-        self.join 
+        self.join
         stream_raise_exception $!
       ensure
-        self.join if self.closed? or self.eof? 
+        self.join if ! @stream_exception && (self.closed? || self.eof?)
       end
     else
       super(*args)
@@ -232,33 +230,55 @@ module ConcurrentStream
   def read(*args)
     begin
       super(*args)
-    rescue
-      raise stream_exception if stream_exception
-      raise $!
+    rescue Exception
+      @stream_exception ||= $!
+      raise @stream_exception
     ensure
-      begin
-        close unless closed?
-      rescue Exception
-        raise $! if ConcurrentStreamProcessFailed === $!
-      end if autojoin && ! closed? && eof?
+      if ! @stream_exception && autojoin && ! closed?
+        begin
+          done = eof?
+        rescue Exception
+          self.abort($!)
+          raise $!
+        end
+        close if done
+      end
     end
   end
 
   def add_callback(&block)
     old_callback = callback
-    @callback = Proc.new do 
+    @callback = Proc.new do
       old_callback.call if old_callback
       block.call
     end
   end
 
   def stream_raise_exception(exception)
+    self.stream_exception = exception
     threads.each do |thread|
       thread.raise exception
     end
-    self.stream_exception = exception
-
     self.abort
   end
 
+  def self.process_stream(stream, close: true, join: true, message: "process_stream", **kwargs, &block)
+    ConcurrentStream.setup(stream, **kwargs)
+    begin
+      begin
+        yield
+      ensure
+        stream.close if close && stream.respond_to?(:close) && ! (stream.respond_to?(:closed?) && stream.closed?)
+        stream.join if join && stream.respond_to?(:join) && ! stream.joined?
+      end
+    rescue Aborted
+      Log.low "Aborted #{message}: #{$!.message}"
+      stream.abort($!) if stream.respond_to?(:abort) && ! stream.aborted?
+      raise $!
+    rescue Exception
+      Log.low "Exception #{message}: #{$!.message}"
+      stream.abort($!) if stream.respond_to?(:abort) && ! stream.aborted?
+      raise $!
+    end
+  end
 end