scout-gear 9.1.0 → 10.1.0

data/scout_commands/workflow/task

@@ -54,10 +54,10 @@ if help
   exit 0
 end
 
+job_inputs = task.get_SOPT
+
 if load_inputs
-  job_inputs = workflow.tasks[task_name].load_inputs(load_inputs)
-else
-  job_inputs = task.get_SOPT
+  job_inputs = job_inputs.merge(workflow.tasks[task_name].load_inputs(load_inputs))
 end
 
 if override_deps

data/test/scout/tsv/test_open.rb

@@ -137,7 +137,7 @@ row2 aa bb cc
     dumper =  TSV::Dumper.new :key_field => "Key", :fields => ["Field"], type: :single
     dumper.init
     assert_raise ScoutException do
-      Log.with_severity 0 do
+      Log.with_severity 7 do
         i = 0
         TSV.traverse lines, :into => dumper, cpus: 3 do |l|
           raise ScoutException if i > 10

data/test/scout/tsv/util/test_sort.rb

@@ -0,0 +1,23 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+
+class TestClass < Test::Unit::TestCase
+
+  def test_sort_by
+    content =<<-EOF
+#ID ValueA ValueB Comment
+row1 a B c
+row2 A b C
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(File.open(filename), :type => :list, :sep => /\s/)
+      assert_equal %w(row2 row1), tsv.sort_by("ValueA").collect{|k,v| k}
+      assert_equal %w(row1 row2), tsv.sort_by("ValueB").collect{|k,v| k}
+    end
+  end
+
+end
+

data/test/scout/workflow/task/test_inputs.rb

@@ -219,14 +219,10 @@ class TestTaskInput < Test::Unit::TestCase
       original_digest =  task.process_inputs(inputs).last
 
       TmpFile.with_file do |save_directory|
-        iii inputs
         task.save_inputs(save_directory, inputs)
         Open.rm(file1)
         new_inputs = task.load_inputs(save_directory)
-        iii new_inputs
         new_digest =  task.process_inputs(new_inputs).last
-        iii new_digest
-        iii original_digest
         assert_equal original_digest, new_digest
       end
     end

metadata

@@ -1,17 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: scout-gear
 version: !ruby/object:Gem::Version
-  version: 9.1.0
+  version: 10.1.0
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-06-30 00:00:00.000000000 Z
+date: 2023-07-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: term-ansicolor
+  name: scout-essentials
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -52,20 +52,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: sys-proctable
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: RubyInline
   requirement: !ruby/object:Gem::Requirement
@@ -159,65 +145,12 @@ files:
 - lib/rbbt-scout.rb
 - lib/scout-gear.rb
 - lib/scout.rb
-- lib/scout/cmd.rb
-- lib/scout/concurrent_stream.rb
-- lib/scout/config.rb
-- lib/scout/exceptions.rb
-- lib/scout/indiferent_hash.rb
-- lib/scout/indiferent_hash/case_insensitive.rb
-- lib/scout/indiferent_hash/options.rb
-- lib/scout/log.rb
-- lib/scout/log/color.rb
-- lib/scout/log/color_class.rb
-- lib/scout/log/fingerprint.rb
-- lib/scout/log/progress.rb
-- lib/scout/log/progress/report.rb
-- lib/scout/log/progress/util.rb
-- lib/scout/log/trap.rb
-- lib/scout/meta_extension.rb
-- lib/scout/misc.rb
-- lib/scout/misc/digest.rb
-- lib/scout/misc/filesystem.rb
-- lib/scout/misc/format.rb
-- lib/scout/misc/helper.rb
-- lib/scout/misc/insist.rb
-- lib/scout/misc/monitor.rb
-- lib/scout/misc/system.rb
-- lib/scout/named_array.rb
 - lib/scout/offsite.rb
+- lib/scout/offsite/exceptions.rb
 - lib/scout/offsite/ssh.rb
 - lib/scout/offsite/step.rb
 - lib/scout/offsite/sync.rb
-- lib/scout/open.rb
-- lib/scout/open/lock.rb
-- lib/scout/open/lock/lockfile.rb
-- lib/scout/open/remote.rb
-- lib/scout/open/stream.rb
-- lib/scout/open/util.rb
-- lib/scout/path.rb
-- lib/scout/path/find.rb
-- lib/scout/path/tmpfile.rb
-- lib/scout/path/util.rb
-- lib/scout/persist.rb
-- lib/scout/persist/open.rb
-- lib/scout/persist/path.rb
-- lib/scout/persist/serialize.rb
-- lib/scout/resource.rb
-- lib/scout/resource/open.rb
-- lib/scout/resource/path.rb
-- lib/scout/resource/produce.rb
-- lib/scout/resource/produce/rake.rb
-- lib/scout/resource/scout.rb
-- lib/scout/resource/software.rb
-- lib/scout/resource/util.rb
 - lib/scout/semaphore.rb
-- lib/scout/simple_opt.rb
-- lib/scout/simple_opt/accessor.rb
-- lib/scout/simple_opt/doc.rb
-- lib/scout/simple_opt/get.rb
-- lib/scout/simple_opt/parse.rb
-- lib/scout/simple_opt/setup.rb
-- lib/scout/tmpfile.rb
 - lib/scout/tsv.rb
 - lib/scout/tsv/attach.rb
 - lib/scout/tsv/change_id.rb
@@ -239,8 +172,10 @@ files:
 - lib/scout/tsv/util/process.rb
 - lib/scout/tsv/util/reorder.rb
 - lib/scout/tsv/util/select.rb
+- lib/scout/tsv/util/sort.rb
 - lib/scout/tsv/util/unzip.rb
 - lib/scout/work_queue.rb
+- lib/scout/work_queue/exceptions.rb
 - lib/scout/work_queue/socket.rb
 - lib/scout/work_queue/worker.rb
 - lib/scout/workflow.rb
@@ -283,49 +218,12 @@ files:
 - share/software/install_helpers
 - share/templates/command
 - share/templates/workflow.rb
-- test/scout/indiferent_hash/test_case_insensitive.rb
-- test/scout/indiferent_hash/test_options.rb
-- test/scout/log/test_color.rb
-- test/scout/log/test_progress.rb
-- test/scout/misc/test_digest.rb
-- test/scout/misc/test_filesystem.rb
-- test/scout/misc/test_insist.rb
-- test/scout/misc/test_system.rb
 - test/scout/offsite/test_ssh.rb
 - test/scout/offsite/test_step.rb
 - test/scout/offsite/test_sync.rb
 - test/scout/offsite/test_task.rb
-- test/scout/open/test_lock.rb
-- test/scout/open/test_remote.rb
-- test/scout/open/test_stream.rb
-- test/scout/open/test_util.rb
-- test/scout/path/test_find.rb
-- test/scout/path/test_util.rb
-- test/scout/persist/test_open.rb
-- test/scout/persist/test_path.rb
-- test/scout/persist/test_serialize.rb
-- test/scout/resource/test_path.rb
-- test/scout/resource/test_produce.rb
-- test/scout/resource/test_software.rb
-- test/scout/resource/test_util.rb
-- test/scout/simple_opt/test_doc.rb
-- test/scout/simple_opt/test_get.rb
-- test/scout/simple_opt/test_parse.rb
-- test/scout/simple_opt/test_setup.rb
-- test/scout/test_cmd.rb
-- test/scout/test_concurrent_stream.rb
-- test/scout/test_config.rb
-- test/scout/test_indiferent_hash.rb
-- test/scout/test_log.rb
-- test/scout/test_meta_extension.rb
-- test/scout/test_misc.rb
-- test/scout/test_named_array.rb
-- test/scout/test_open.rb
-- test/scout/test_path.rb
-- test/scout/test_persist.rb
-- test/scout/test_resource.rb
+- test/scout/test_offsite.rb
 - test/scout/test_semaphore.rb
-- test/scout/test_tmpfile.rb
 - test/scout/test_tsv.rb
 - test/scout/test_work_queue.rb
 - test/scout/test_workflow.rb
@@ -347,6 +245,7 @@ files:
 - test/scout/tsv/util/test_process.rb
 - test/scout/tsv/util/test_reorder.rb
 - test/scout/tsv/util/test_select.rb
+- test/scout/tsv/util/test_sort.rb
 - test/scout/tsv/util/test_unzip.rb
 - test/scout/work_queue/test_socket.rb
 - test/scout/work_queue/test_worker.rb

data/lib/scout/cmd.rb

@@ -1,347 +0,0 @@
-require_relative 'indiferent_hash'
-require_relative 'concurrent_stream'
-require_relative 'log'
-require_relative 'open/stream'
-require 'stringio'
-require 'open3'
-
-module CMD
-
-  TOOLS = IndiferentHash.setup({})
-  def self.tool(tool, claim = nil, test = nil, cmd = nil, &block)
-    TOOLS[tool] = [claim, test, block, cmd]
-  end
-
-  def self.conda(tool, env = nil, channel = 'bioconda')
-    if env
-      CMD.cmd("bash -l -c '(conda activate #{env} && conda install #{tool} -c #{channel})'")
-    else
-      CMD.cmd("bash -l -c 'conda install #{tool} -c #{channel}'")
-    end
-  end
-
-
-  def self.get_tool(tool)
-    return tool.to_s unless TOOLS[tool]
-
-    @@init_cmd_tool ||= IndiferentHash.setup({})
-
-    claim, test, block, cmd = TOOLS[tool]
-    cmd = tool.to_s if cmd.nil?
-
-    if !@@init_cmd_tool[tool]
-
-      begin
-        if test
-          CMD.cmd(test + " ")
-        else
-          CMD.cmd("#{cmd} --help")
-        end
-      rescue
-        if claim
-          claim.produce
-        else
-          block.call
-        end
-      end
-      version_txt = ""
-      version = nil
-      ["--version", "-version", "--help", ""].each do |f|
-        begin
-          version_txt += CMD.cmd("#{cmd} #{f} 2>&1", :nofail => true).read
-          version = CMD.scan_version_text(version_txt, tool)
-          break if version
-        rescue
-          Log.exception $!
-        end
-      end
-
-      @@init_cmd_tool[tool] = version || true
-
-      return cmd if cmd
-    end
-
-    cmd
-  end
-  def self.scan_version_text(text, cmd = nil)
-    cmd = "NOCMDGIVE" if cmd.nil? || cmd.empty?
-    text = Misc.fixutf8 text
-    text.split("\n").each do |line|
-      next unless line =~ /\W#{cmd}\W/i
-      m = line.match(/(v(?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
-      return m[1] if m
-      m = line.match(/((?:\d+\.)*\d+(?:-[a-z_]+)?v)/i)
-      return m[1] if m
-      next unless line =~ /\Wversion\W/i
-      m = line.match(/((?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
-      return m[1] if m
-    end
-    m = text.match(/(?:version.*?|#{cmd}.*?|#{cmd.to_s.split(/[-_.]/).first}.*?|v)((?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
-    return m[1] if m
-    m = text.match(/(?:#{cmd}.*(v.*|.*v))/i)
-    return m[1] if m
-    nil
-  end
-  def self.versions
-    return {} unless defined? @@init_cmd_tool
-    @@init_cmd_tool.select{|k,v| v =~ /\d+\./ }
-  end
-
-  def self.bash(cmd)
-    cmd = %Q(bash <<EOF\n#{cmd}\nEOF\n)
-    CMD.cmd(cmd, :autojoin => true)
-  end
-
-  def self.process_cmd_options(options = {})
-    add_dashes = IndiferentHash.process_options options, :add_option_dashes
-
-    string = ""
-    options.each do |option, value|
-      raise "Invalid option key: #{option.inspect}" if option.to_s !~ /^[a-z_0-9\-=.]+$/i
-      #raise "Invalid option value: #{value.inspect}" if value.to_s.include? "'"
-      value = value.gsub("'","\\'") if value.to_s.include? "'"
-
-      option = "--" << option.to_s if add_dashes and option.to_s[0] != '-'
-
-      case
-      when value.nil? || FalseClass === value
-        next
-      when TrueClass === value
-        string << "#{option} "
-      else
-        if option.to_s.chars.to_a.last == "="
-          string << "#{option}'#{value}' "
-        else
-          string << "#{option} '#{value}' "
-        end
-      end
-    end
-
-    string.strip
-  end
-
-  def self.cmd(tool, cmd = nil, options = {}, &block)
-    options, cmd = cmd, nil if Hash === cmd
-
-    options    = IndiferentHash.add_defaults options, :stderr => Log::DEBUG
-    in_content = options.delete(:in)
-    stderr     = options.delete(:stderr)
-    post       = options.delete(:post)
-    pipe       = options.delete(:pipe)
-    log        = options.delete(:log)
-    no_fail    = options.delete(:no_fail)
-    no_fail    = options.delete(:nofail) if no_fail.nil?
-    no_wait    = options.delete(:no_wait)
-    xvfb       = options.delete(:xvfb)
-    bar        = options.delete(:progress_bar)
-    save_stderr = options.delete(:save_stderr)
-    autojoin   = options.delete(:autojoin)
-    autojoin   = no_wait if autojoin.nil?
-
-    dont_close_in  = options.delete(:dont_close_in)
-
-    log = true if log.nil?
-
-    if cmd.nil? && ! Symbol === tool
-      cmd = tool
-    else
-      tool = get_tool(tool)
-      if cmd.nil?
-        cmd = tool
-      else
-        cmd = tool + ' ' + cmd
-      end
-
-    end
-
-    case xvfb
-    when TrueClass
-      cmd = "xvfb-run --server-args='-screen 0 1024x768x24' --auto-servernum #{cmd}"
-    when String
-      cmd = "xvfb-run --server-args='#{xvfb}' --auto-servernum --server-num=1 #{cmd}"
-    when String
-    end
-
-    if stderr == true
-      stderr = Log::HIGH
-    end
-
-    cmd_options = process_cmd_options options
-    if cmd =~ /'\{opt\}'/
-      cmd.sub!('\'{opt}\'', cmd_options)
-    else
-      cmd << " " << cmd_options
-    end
-
-    in_content = StringIO.new in_content if String === in_content
-
-    sin, sout, serr, wait_thr = begin
-                                  Open3.popen3(ENV, cmd)
-                                rescue
-                                  Log.warn $!.message
-                                  raise ProcessFailed, nil, cmd unless no_fail
-                                  return
-                                end
-    pid = wait_thr.pid
-
-    Log.debug{"CMD: [#{pid}] #{cmd}".strip if log}
-
-    if in_content.respond_to?(:read)
-      in_thread = Thread.new(Thread.current) do |parent|
-        begin
-          Thread.current.report_on_exception = false if no_fail
-          Thread.current["name"] = "CMD in"
-          while c = in_content.read(Open::BLOCK_SIZE)
-            sin << c
-          end
-          sin.close  unless sin.closed?
-
-          unless dont_close_in
-            in_content.close unless in_content.closed?
-            in_content.join if in_content.respond_to? :join
-          end
-        rescue
-          Log.error "Error in CMD  [#{pid}] #{cmd}: #{$!.message}" unless no_fail
-          raise $!
-        end
-      end
-      Thread.pass until in_thread["name"]
-    else
-      in_thread = nil
-      sin.close
-    end
-
-    pids = [pid]
-
-    if pipe
-
-      ConcurrentStream.setup sout, :pids => pids, :autojoin => autojoin, :no_fail => no_fail
-
-      sout.callback = post if post
-
-      if (Integer === stderr and log) || bar
-        err_thread = Thread.new do
-          Thread.current["name"] = "Error log: [#{pid}] #{ cmd }"
-          begin
-          while line = serr.gets
-            bar.process(line) if bar
-            sout.log = line
-            sout.std_err << line if save_stderr
-            Log.log "STDERR [#{pid}]: " +  line, stderr if log
-          end
-          serr.close
-          rescue
-            Log.exception $!
-            raise $!
-          end
-        end
-      else
-        err_thread = Open.consume_stream(serr, true)
-      end
-
-      sout.threads = [in_thread, err_thread, wait_thr].compact
-
-      sout
-    else
-
-      if bar
-        err = ""
-        err_thread = Thread.new do
-          while not serr.eof?
-            line = serr.gets
-            bar.process(line)
-            err << line if Integer === stderr and log
-          end
-          serr.close
-        end
-      elsif log and Integer === stderr
-        err = ""
-        err_thread = Thread.new do
-          while not serr.eof?
-            err << serr.gets
-          end
-          serr.close
-        end
-      else
-        Open.consume_stream(serr, true)
-        err_thread = nil
-        err = ""
-      end
-
-      ConcurrentStream.setup sout, :pids => pids, :threads => [in_thread, err_thread].compact, :autojoin => autojoin, :no_fail => no_fail
-
-      begin
-        out = StringIO.new sout.read
-        sout.close unless sout.closed?
-
-        status = wait_thr.value
-        if status && ! status.success? && ! no_fail
-          if !err.empty?
-            raise ProcessFailed.new pid, "#{cmd} failed with error status #{status.exitstatus}.\n#{err}"
-          else
-            raise ProcessFailed.new pid, "#{cmd} failed with error status #{status.exitstatus}"
-          end
-        else
-          Log.log err, stderr if Integer === stderr and log
-        end
-        out
-      ensure
-        post.call if post
-      end
-    end
-  end
-
-  def self.cmd_pid(*args)
-    all_args = *args
-
-    bar = all_args.last[:progress_bar] if Hash === all_args.last
-
-    all_args << {} unless Hash === all_args.last
-
-    level = all_args.last[:log] || 0
-    level = 0 if TrueClass === level
-    level = 10 if FalseClass === level
-    level = level.to_i
-
-    all_args.last[:log] = true
-    all_args.last[:pipe] = true
-
-    io = cmd(*all_args)
-    pid = io.pids.first
-
-    line = "" if bar
-    starting = true
-    while c = io.getc
-      if starting
-        if pid
-          Log.logn "STDOUT [#{pid}]: ", level
-        else
-          Log.logn "STDOUT: ", level
-        end
-        starting = false
-      end
-      STDERR << c if Log.severity <= level
-      line << c if bar
-      if c == "\n"
-        bar.process(line) if bar
-        starting = true
-        line = "" if bar
-      end
-    end
-    begin
-      io.join
-      bar.remove if bar
-    rescue
-      bar.remove(true) if bar
-      raise $!
-    end
-
-    nil
-  end
-
-  def self.cmd_log(*args)
-    cmd_pid(*args)
-    nil
-  end
-
-end