scout-gear 7.3.0 → 8.1.0

data/lib/scout/meta_extension.rb

@@ -2,7 +2,7 @@ module MetaExtension
   def self.extended(base)
     meta = class << base; self; end
 
-    base.class_variable_set("@@extension_attrs", [])
+    base.class_variable_set("@@extension_attrs", []) unless base.class_variables.include?("@@extension_attrs")
 
     meta.define_method(:extension_attr) do |*attrs|
       self.class_variable_get("@@extension_attrs").concat attrs
@@ -11,6 +11,14 @@ module MetaExtension
       end
     end
 
+    meta.define_method(:extended) do |obj|
+      attrs = self.class_variable_get("@@extension_attrs")
+
+      obj.instance_variable_set(:@extension_attrs, []) unless obj.instance_variables.include?(:@extension_attrs)
+      extension_attrs = obj.instance_variable_get(:@extension_attrs)
+      extension_attrs.concat attrs
+    end
+
     meta.define_method(:setup) do |*args,&block|
       if block_given?
         obj, rest = block, args
@@ -18,7 +26,8 @@ module MetaExtension
         obj, *rest = args
       end
       obj = block if obj.nil?
-      obj.extend base
+      obj.extend base unless base === obj
+
       attrs = self.class_variable_get("@@extension_attrs")
 
       return if attrs.nil? || attrs.empty?
@@ -27,8 +36,6 @@ module MetaExtension
           ((! (rlkey = rlast.keys.first).nil? && attrs.include?(rlkey.to_sym)) ||
            (! attrs.length != 1 ))
 
-          
-
         pairs = rlast
       else
         pairs = attrs.zip(rest)
@@ -43,17 +50,51 @@ module MetaExtension
 
     base.define_method(:extension_attr_hash) do 
       attr_hash = {}
-      meta.class_variable_get("@@extension_attrs").each do |name|
+      @extension_attrs.each do |name|
         attr_hash[name] = self.instance_variable_get("@#{name}")
       end
       attr_hash
     end
 
     base.define_method(:annotate) do |other|
-      attr_values = meta.class_variable_get("@@extension_attrs").collect do |a|
+      attr_values = @extension_attrs.collect do |a|
         self.instance_variable_get("@#{a}")
       end
       base.setup(other, *attr_values)
     end
+
+    base.define_method(:purge) do
+      new = self.dup
+
+      if new.instance_variables.include?(:@extension_attrs)
+        new.instance_variable_get(:@extension_attrs).each do |a|
+          new.remove_instance_variable("@#{a}")
+        end
+        new.remove_instance_variable("@extension_attrs")
+      end
+
+      new
+    end
+  end
+
+  def self.is_extended?(obj)
+    obj.respond_to?(:extension_attr_hash)
+  end
+
+  def self.purge(obj)
+    case obj
+    when nil
+      nil
+    when Array
+      obj.collect{|e| purge(e) }
+    when Hash
+      new = {}
+      obj.each do |k,v|
+        new[k] = purge(v)
+      end
+      new
+    else
+      is_extended?(obj) ? obj.purge : obj
+    end
   end
 end

data/lib/scout/misc/digest.rb

@@ -1,4 +1,5 @@
 module Misc
+  MAX_ARRAY_DIGEST_LENGTH = 100_000
   def self.digest_str(obj)
     if obj.respond_to?(:digest_str)
       obj.digest_str
@@ -9,12 +10,19 @@ module Misc
         if Path === obj || ! Open.exists?(obj)
           '\'' << obj << '\''
         else
-          Misc.file_md5(obj)
+          "File MD5: #{Misc.file_md5(obj)}"
         end
       when Integer, Symbol
         obj.to_s
       when Array
-        '[' << obj.inject(""){|acc,o| acc.empty? ? Misc.digest_str(o) : acc << ', ' << Misc.digest_str(o) } << ']'
+        if obj.length > MAX_ARRAY_DIGEST_LENGTH
+          length = obj.length
+          mid = length/2
+          sample_pos = [1, 2, mid, length-2, length-1]
+          "[#{length}:" << obj.values_at(*sample_pos).inject(""){|acc,o| acc.empty? ? Misc.digest_str(o) : acc << ', ' << Misc.digest_str(o) } << ']'
+        else
+          '[' << obj.inject(""){|acc,o| acc.empty? ? Misc.digest_str(o) : acc << ', ' << Misc.digest_str(o) } << ']'
+        end
       when Hash
         '{' << obj.inject(""){|acc,p| s = Misc.digest_str(p.first) << "=" << Misc.digest_str(p.last); acc.empty? ? s : acc << ', ' << s } << '}'
       when Integer
@@ -40,12 +48,13 @@ module Misc
   end
 
   def self.digest(obj)
-    str = Misc.digest_str(obj)
+    str = String === obj ? obj : Misc.digest_str(obj)
     Digest::MD5.hexdigest(str)
   end
 
   def self.file_md5(file)
     file = file.find if Path === file
+    file = File.expand_path(file)
     #md5file = file + '.md5'
     Persist.persist("MD5:#{file}", :string) do
       Digest::MD5.file(file).hexdigest

data/lib/scout/misc/format.rb

@@ -25,11 +25,23 @@ module Misc
     str
   end
 
+  CHAR_SENCONDS = ENV["SCOUT_NOCOLOR"] == "true" ? "sec" : "″"
+  def self.format_seconds_short(time)
+    if time < 0.0001
+      "%.5g" % time + CHAR_SENCONDS
+    elsif time < 60
+      "%.2g" % time + CHAR_SENCONDS
+    else
+      format_seconds(time)
+    end
+  end
+
+
   
-  MAX_WIDTH = 100
+  MAX_TTY_LINE_WIDTH = 100
   def self.format_paragraph(text, size = nil, indent = nil, offset = nil)
-    size ||= Log.tty_size || MAX_WIDTH
-    size = MAX_WIDTH if size > MAX_WIDTH
+    size ||= Log.tty_size || MAX_TTY_LINE_WIDTH
+    size = MAX_TTY_LINE_WIDTH if size > MAX_TTY_LINE_WIDTH
     indent ||= 0
     offset ||= 0
 
@@ -63,9 +75,14 @@ module Misc
   end
 
   def self.format_definition_list_item(dt, dd,  indent = nil, size = nil, color = :yellow)
-    size ||= Log.tty_size || MAX_WIDTH
-    size = MAX_WIDTH if size > MAX_WIDTH
-    indent ||= size / 3
+    if size.nil?
+      base_size = MAX_TTY_LINE_WIDTH
+      base_indent = indent || (base_size / 3)
+      size = base_size - base_indent
+    end
+
+    indent ||= base_indent || size / 3
+
     dd = "" if dd.nil?
     dt = Log.color color, dt if color
     dt = dt.to_s  unless dd.empty?
@@ -84,8 +101,8 @@ module Misc
   end
 
   def self.format_definition_list(defs, indent = nil, size = nil, color = :yellow, sep = "\n\n")
-    size ||= Log.tty_size || MAX_WIDTH
     indent ||= 30
+    size ||= (Log.tty_size || MAX_TTY_LINE_WIDTH) - indent
     entries = []
     defs.each do |dt,dd|
       text = format_definition_list_item(dt,dd,indent, size,color)

data/lib/scout/misc/insist.rb

@@ -32,7 +32,7 @@ module Misc
       end
       raise $!
     rescue Exception
-      Log.exception $! if ENV["RBBT_LOG_INSIST"] == 'true'
+      Log.exception $! if ENV["SCOUT_LOG_INSIST"] == 'true'
       if msg
         Log.warn("Insisting after exception: #{$!.class} #{$!.message} -- #{msg}")
       elsif FalseClass === msg

data/lib/scout/misc/monitor.rb

@@ -2,6 +2,7 @@ module Misc
   def self.pid_alive?(pid)
     !! Process.kill(0, pid) rescue false
   end
+
   def self.benchmark(repeats = 1, message = nil)
     require 'benchmark'
     res = nil
@@ -41,4 +42,25 @@ module Misc
 
     res
   end
+
+  def self.exec_time(&block)
+    start = Time.now
+    eend = nil
+    begin
+      yield
+    ensure
+      eend = Time.now
+    end
+    eend - start
+  end
+
+  def self.wait_for_interrupt
+    while true
+      begin
+        sleep 1
+      rescue Interrupt
+        break
+      end
+    end
+  end
 end

data/lib/scout/misc/system.rb

@@ -1,4 +1,18 @@
+require 'sys/proctable'
+
 module Misc
+
+  def self.hostname
+    @@hostname ||= begin
+                     `hostname`.strip
+                   end
+  end
+
+  def self.children(ppid = nil)
+    ppid ||= Process.pid
+    Sys::ProcTable.ps.select{ |pe| pe.ppid == ppid }
+  end
+
   def self.env_add(var, value, sep = ":", prepend = true)
     if ENV[var].nil?
       ENV[var] = value
@@ -12,4 +26,48 @@ module Misc
       end
     end
   end
+
+  def self.with_env(var, value, &block)
+    old_value = ENV[var]
+    begin
+      ENV[var] = value
+      yield
+    ensure
+      ENV[var] = old_value
+    end
+  end
+  
+  def self.update_git(gem_name = 'scout-gear')
+    gem_name = 'scout-gear' if gem_name.nil?
+    dir = File.join(__dir__, '../../../../', gem_name)
+    return unless Open.exist?(dir)
+    Misc.in_dir dir do
+      begin
+        begin
+          CMD.cmd_log('git pull')
+        rescue
+          raise "Could not update #{gem_name}"
+        end
+
+        begin
+          CMD.cmd_log('git submodule update')
+        rescue
+          raise "Could not update #{gem_name} submodules"
+        end
+
+
+        begin
+          CMD.cmd_log('rake install')
+        rescue
+          raise "Could not install updated #{gem_name}"
+        end
+      rescue
+        Log.warn $!.message
+      end
+    end
+  end
+
+  def self.processors
+    Etc.nprocessors
+  end
 end

data/lib/scout/named_array.rb

@@ -1,13 +1,25 @@
 require_relative 'meta_extension'
 module NamedArray
   extend MetaExtension
-  extension_attr :fields
+  extension_attr :fields, :key
 
-  def self.identify_name(names, selected)
+  def self.field_match(field, name)
+    if (String === field) && (String === name)
+      field == name || 
+        field.start_with?(name) || field.include?("(" + name + ")") ||
+        name.start_with?(field) || name.include?("(" + field + ")")
+    else
+      field == name
+    end
+  end
+
+  def self.identify_name(names, selected, strict: false)
     res = (Array === selected ? selected : [selected]).collect do |field|
       case field
       when nil
         0
+      when Range
+        field
       when Integer
         field
       when Symbol
@@ -26,7 +38,8 @@ module NamedArray
         if field =~ /^\d+$/
           next identify_names(names, field.to_i)
         end
-        pos = names.index{|name| name.start_with?(field) }
+        next pos if strict
+        pos = names.index{|name| field_match(field, name) }
         next pos if pos
         nil
       else
@@ -37,6 +50,10 @@ module NamedArray
     Array === selected ? res : res.first
   end
 
+  def identify_name(selected)
+    NamedArray.identify_name(fields, selected)
+  end
+
   def positions(fields)
     if Array ==  fields
       fields.collect{|field|
@@ -49,6 +66,7 @@ module NamedArray
 
   def [](key)
     pos = NamedArray.identify_name(@fields, key)
+    return nil if pos.nil?
     super(pos)
   end
 
@@ -65,4 +83,56 @@ module NamedArray
     end
     IndiferentHash.setup hash
   end
+
+  def values_at(*positions)
+    super(*identify_name(positions))
+  end
+
+  def self._zip_fields(array, max = nil)
+    return [] if array.nil? or array.empty? or (first = array.first).nil?
+
+    max = array.collect{|l| l.length }.max if max.nil?
+
+    rest = array[1..-1].collect{|v|
+      v.length == 1 & max > 1 ? v * max : v
+    }
+
+    first = first * max if first.length == 1 and max > 1
+
+    first.zip(*rest)
+  end
+
+  def self.zip_fields(array)
+    if array.length < 10000
+      _zip_fields(array)
+    else
+      zipped_slices = []
+      max = array.collect{|l| l.length}.max
+      array.each_slice(10000) do |slice|
+        zipped_slices << _zip_fields(slice, max)
+      end
+      new = zipped_slices.first
+      zipped_slices[1..-1].each do |rest|
+        rest.each_with_index do |list,i|
+          new[i].concat list
+        end
+      end
+      new
+    end
+  end
+
+  def self.add_zipped(source, new)
+    source.zip(new).each do |s,n|
+      s.concat(n)
+    end
+    source
+  end
+
+  def method_missing(name, *args)
+    if identify_name(name)
+      return self[name]
+    else
+      return super(name, *args)
+    end
+  end
 end

data/lib/scout/offsite/ssh.rb

@@ -0,0 +1,171 @@
+require 'net/ssh'
+
+class SSHLine
+  class << self
+    attr_accessor :default_server
+    def default_server
+      @@default_server ||= begin
+                             ENV["SCOUT_OFFSITE"] || ENV["SCOUT_SERVER"] || 'localhost'
+                           end
+    end
+  end
+
+  def initialize(host = :default, user = nil)
+    host = SSHLine.default_server if host.nil? || host == :default
+    @host = host
+    @user = user
+
+    @ssh = Net::SSH.start(@host, @user)
+
+    @ch = @ssh.open_channel do |ch|
+      ch.exec 'bash -l'
+    end
+
+    @ch.on_data do |_,data|
+      if m = data.match(/DONECMD: (\d+)\n/)
+        @exit_status = m[1].to_i
+        @output << data.sub(m[0],'')
+        serve_output 
+      else
+        @output << data
+      end
+    end
+
+    @ch.on_extended_data do |_,c,err|
+      STDERR.write err 
+    end
+  end
+
+
+  def self.reach?(server = SSHLine.default_server)
+    Persist.memory(server, :key => "Reach server") do
+      begin
+        CMD.cmd("ssh #{server} bash -l -c \"scout\"") 
+        true
+      rescue Exception
+        false
+      end
+    end
+  end
+
+  def send_cmd(command)
+    @output = ""
+    @complete_output = false
+    @ch.send_data(command+"\necho DONECMD: $?\n")
+  end
+
+  def serve_output
+    @complete_output = true
+  end
+
+  def run(command)
+    send_cmd(command)
+    @ssh.loop{ ! @complete_output}
+    if @exit_status.to_i == 0
+      return @output
+    else
+      raise SSHProcessFailed.new @host, command
+    end
+  end
+
+  def ruby(script)
+    @output = ""
+    @complete_output = false
+    cmd = "ruby -e \"#{script.gsub('"','\\"')}\"\n"
+    Log.debug "Running ruby on #{@host}:\n#{ script }"
+    @ch.send_data(cmd)
+    @ch.send_data("echo DONECMD: $?\n")
+    @ssh.loop{ !@complete_output }
+    if @exit_status.to_i == 0
+      return @output
+    else
+      raise SSHProcessFailed.new @host, "Ruby script:\n#{script}"
+    end
+  end
+
+  def scout(script)
+    scout_script =<<-EOF
+require 'scout'
+SSHLine.run_local do
+#{script.strip}
+end
+    EOF
+
+    m = ruby(scout_script)
+    Marshal.load m
+  end
+
+  def workflow(workflow, script)
+    preamble =<<-EOF
+wf = Workflow.require_workflow('#{workflow}')
+    EOF
+
+    scout(preamble + "\n" + script)
+  end
+
+  class Mock < SSHLine
+    def initialize
+    end
+
+    def run(command)
+      CMD.cmd(command)
+    end
+
+    def ruby(script)
+      cmd = "ruby -e \"#{script.gsub('"','\\"')}\"\n"
+      CMD.cmd(cmd)
+    end
+  end
+
+  @connections = {}
+  def self.open(host, user = nil)
+    @connections[[host, user]] ||=
+      begin
+        if host == 'localhost'
+          SSHLine::Mock.new
+        else
+          SSHLine.new host, user
+        end
+      end
+  end
+
+  def self.run(server, cmd, options = nil)
+    cmd = cmd * " " if Array === cmd
+    cmd += " " + CMD.process_cmd_options(options) if options
+    open(server).run(cmd)
+  end
+
+  def self.ruby(server, script)
+    open(server).ruby(script)
+  end
+
+  def self.scout(server, script)
+    open(server).scout(script)
+  end
+
+  def self.workflow(server, workflow, script)
+    open(server).workflow(workflow, script)
+  end
+
+  def self.command(server, command, argv = [], options = nil)
+    command = "scout #{command}" unless command && command.include?('scout')
+    argv_str = (argv - ["--"]).collect{|v| '"' + v.to_s + '"' } * " "
+    command = "#{command} #{argv_str}"
+    Log.debug "Offsite #{server} running: #{command}"
+    run(server, command, options)
+  end
+
+  def self.mkdir(server, path)
+    self.run server, "mkdir -p '#{path}'"
+  end
+
+  def self.run_local(&block)
+    res = begin
+            old_stdout = STDOUT.dup; STDOUT.reopen(STDERR)
+            block.call
+          ensure
+            STDOUT.reopen(old_stdout)
+          end
+    puts Marshal.dump(res)
+  end
+end

data/lib/scout/offsite/step.rb

@@ -0,0 +1,83 @@
+require_relative '../workflow/step'
+require_relative 'ssh'
+require_relative 'sync'
+
+module OffsiteStep
+
+  extend MetaExtension
+  extension_attr :server, :workflow_name, :clean_id, :provided_inputs
+
+  def inputs_directory
+    @inputs_directory ||= begin
+                            if provided_inputs && provided_inputs.any?
+                              file = ".scout/tmp/step_inputs/#{workflow}/#{task_name}/#{name}"
+                              TmpFile.with_path do |inputs_dir|
+                                task.save_inputs(inputs_dir, provided_inputs)
+                                SSHLine.rsync(inputs_dir, file, target: server, directory: true)
+                              end
+                              file
+                            end
+                          end
+  end
+
+  def workflow_name
+    @workflow_name || workflow.to_s
+  end
+
+  def offsite_job_ssh(script)
+    parts = []
+    parts << <<~EOF.strip
+wf = Workflow.require_workflow "#{workflow_name}";
+    EOF
+
+    if inputs_directory
+      parts << <<~EOF.strip
+job = wf.job(:#{task_name}, "#{clean_name}", :load_inputs => "#{inputs_directory}");
+      EOF
+    else
+      parts << <<~EOF.strip
+job = wf.job(:#{task_name}, "#{clean_name}");
+      EOF
+    end
+
+    parts << script
+
+
+    SSHLine.scout server, parts * "\n"
+  end
+
+  def offsite_path
+    @path = offsite_job_ssh <<~EOF
+      job.path.identify
+    EOF
+  end
+
+  def info
+    info = @info ||= offsite_job_ssh <<~EOF
+    info = Open.exists?(job.info_file) ? job.info : {}
+    info[:running] = true if job.running?
+    info
+    EOF
+
+    @info = nil unless %w(done aborted error).include?(info[:status].to_s)
+
+    info
+  end
+
+  def done?
+    status == :done
+  end
+
+  def exec
+    bundle_files = offsite_job_ssh <<~EOF
+    job.run
+    job.bundle_files
+    EOF
+    SSHLine.sync(bundle_files, source: server)
+    self.load
+  end
+
+  def run
+    exec
+  end
+end