scout-gear 8.0.0 → 8.1.0

data/lib/scout/log.rb

@@ -12,9 +12,9 @@ module Log
   end
 
   SEVERITY_NAMES ||= begin
-                       names = %w(DEBUG LOW MEDIUM HIGH INFO WARN ERROR NONE ) 
+                       names = %w(DEBUG LOW MEDIUM HIGH INFO WARN ERROR NONE )
                        names.each_with_index do |name,i|
-                         eval "#{ name } = #{ i }" 
+                         eval "#{ name } = #{ i }"
                        end
                        names
                      end
@@ -32,14 +32,14 @@ module Log
     @@default_severity
   end
 
-  case ENV['SCOUT_LOG'] 
-  when 'DEBUG' 
+  case ENV['SCOUT_LOG']
+  when 'DEBUG'
     self.severity = DEBUG
-  when 'LOW' 
+  when 'LOW'
     self.severity = LOW
-  when 'MEDIUM' 
+  when 'MEDIUM'
     self.severity = MEDIUM
-  when 'HIGH' 
+  when 'HIGH'
     self.severity = HIGH
   when nil
     self.severity = default_severity
@@ -50,23 +50,27 @@ module Log
 
 
   def self.tty_size
-    @tty_size = begin
-                  IO.console.winsize.last
-                rescue
-                  begin
-                    `tput li`
-                  rescue
-                    80
-                  end
-                end
+    @@tty_size ||= Log.ignore_stderr do
+      begin
+        IO.console.winsize.last
+      rescue Exception
+        begin
+          res = `tput li`
+          res = nil if res == ""
+          res || ENV["TTY_SIZE"] || 80
+        rescue Exception
+          ENV["TTY_SIZE"] || 80
+        end
+      end
+    end
   end
 
 
   def self.last_caller(stack)
     line = nil
     pos ||= 0
-    while line.nil? or line =~ /scout\/log\.rb/ and stack.any? 
-      line = stack.shift 
+    while line.nil? or line =~ /scout\/log\.rb/ and stack.any?
+      line = stack.shift
     end
     line ||= caller.first
     line.gsub('`', "'")
@@ -129,9 +133,36 @@ module Log
     out.puts Log.return_line << " " * (Log.tty_size || 80) << Log.return_line unless nocolor
   end
 
+  MUTEX = Mutex.new
+  def self.log_write(str)
+    MUTEX.synchronize do
+      if logfile.nil?
+        begin
+          STDERR.write str
+        rescue
+        end
+      else
+        logfile.write str
+      end
+    end
+  end
+
+  def self.log_puts(str)
+    MUTEX.synchronize do
+      if logfile.nil?
+        begin
+          STDERR.puts str
+        rescue
+        end
+      else
+        logfile.puts str
+      end
+    end
+  end
+
   LAST = "log"
   def self.logn(message = nil, severity = MEDIUM, &block)
-    return if severity < self.severity 
+    return if severity < self.severity
     message ||= block.call if block_given?
     return if message.nil?
 
@@ -147,17 +178,14 @@ module Log
     message = "" << highlight << message << color(0) if severity >= INFO
     str = prefix << " " << message.to_s
 
-    if logfile.nil?
-      STDERR.write str
-    else
-      logfile.write str 
-    end
+    log_write str
+
     Log::LAST.replace "log"
     nil
   end
 
   def self.log(message = nil, severity = MEDIUM, &block)
-    return if severity < self.severity 
+    return if severity < self.severity
     message ||= block.call if block_given?
     return if message.nil?
     message = message + "\n" unless message[-1] == "\n"
@@ -250,9 +278,9 @@ module Log
   end
 
   def self.tsv(tsv, example = false)
-    STDERR.puts Log.color :magenta, "TSV log: " << Log.last_caller(caller).gsub('`',"'")
-    STDERR.puts Log.color(:blue, "=> "<< Log.fingerprint(tsv), true) 
-    STDERR.puts Log.color(:cyan, "=> " << tsv.summary)
+    log_puts Log.color :magenta, "TSV log: " << Log.last_caller(caller).gsub('`',"'")
+    log_puts Log.color(:blue, "=> "<< Log.fingerprint(tsv), true)
+    log_puts Log.color(:cyan, "=> " << tsv.summary)
     if example && ! tsv.empty?
       key = case example
             when TrueClass, :first, "first"
@@ -266,11 +294,11 @@ module Log
       values = tsv[key]
       values = [values] if tsv.type == :flat || tsv.type == :single
       if values.nil?
-        STDERR.puts Log.color(:blue, "Key (#{tsv.key_field}) not present: ") + key
+        log_puts Log.color(:blue, "Key (#{tsv.key_field}) not present: ") + key
       else
-        STDERR.puts Log.color(:blue, "Key (#{tsv.key_field}): ") + key
+        log_puts Log.color(:blue, "Key (#{tsv.key_field}): ") + key
         tsv.fields.zip(values).each do |field,value|
-          STDERR.puts Log.color(:magenta, field + ": ") + (Array === value ? value * ", " : value.to_s)
+          log_puts Log.color(:magenta, field + ": ") + (Array === value ? value * ", " : value.to_s)
         end
       end
     end
@@ -278,14 +306,14 @@ module Log
 
   def self.stack(stack)
     if ENV["SCOUT_ORIGINAL_STACK"] == 'true'
-      STDERR.puts Log.color :magenta, "Stack trace [#{Process.pid}]: " << Log.last_caller(caller)
+      log_puts Log.color :magenta, "Stack trace [#{Process.pid}]: " << Log.last_caller(caller)
       color_stack(stack).each do |line|
-        STDERR.puts line
+        log_puts line
       end
     else
-      STDERR.puts Log.color :magenta, "Stack trace [#{Process.pid}]: " << Log.last_caller(caller)
+      log_puts Log.color :magenta, "Stack trace [#{Process.pid}]: " << Log.last_caller(caller)
       color_stack(stack.reverse).each do |line|
-        STDERR.puts line
+        log_puts line
       end
     end
   end
@@ -293,7 +321,7 @@ module Log
   def self.count_stack
     if ! $count_stacks
       Log.debug "Counting stacks at: " << caller.first
-      return 
+      return
     end
     $stack_counts ||= {}
     head = $count_stacks_head

data/lib/scout/meta_extension.rb

@@ -62,5 +62,39 @@ module MetaExtension
       end
       base.setup(other, *attr_values)
     end
+
+    base.define_method(:purge) do
+      new = self.dup
+
+      if new.instance_variables.include?(:@extension_attrs)
+        new.instance_variable_get(:@extension_attrs).each do |a|
+          new.remove_instance_variable("@#{a}")
+        end
+        new.remove_instance_variable("@extension_attrs")
+      end
+
+      new
+    end
+  end
+
+  def self.is_extended?(obj)
+    obj.respond_to?(:extension_attr_hash)
+  end
+
+  def self.purge(obj)
+    case obj
+    when nil
+      nil
+    when Array
+      obj.collect{|e| purge(e) }
+    when Hash
+      new = {}
+      obj.each do |k,v|
+        new[k] = purge(v)
+      end
+      new
+    else
+      is_extended?(obj) ? obj.purge : obj
+    end
   end
 end

data/lib/scout/misc/digest.rb

@@ -1,4 +1,5 @@
 module Misc
+  MAX_ARRAY_DIGEST_LENGTH = 100_000
   def self.digest_str(obj)
     if obj.respond_to?(:digest_str)
       obj.digest_str
@@ -9,12 +10,19 @@ module Misc
         if Path === obj || ! Open.exists?(obj)
           '\'' << obj << '\''
         else
-          Misc.file_md5(obj)
+          "File MD5: #{Misc.file_md5(obj)}"
         end
       when Integer, Symbol
         obj.to_s
       when Array
-        '[' << obj.inject(""){|acc,o| acc.empty? ? Misc.digest_str(o) : acc << ', ' << Misc.digest_str(o) } << ']'
+        if obj.length > MAX_ARRAY_DIGEST_LENGTH
+          length = obj.length
+          mid = length/2
+          sample_pos = [1, 2, mid, length-2, length-1]
+          "[#{length}:" << obj.values_at(*sample_pos).inject(""){|acc,o| acc.empty? ? Misc.digest_str(o) : acc << ', ' << Misc.digest_str(o) } << ']'
+        else
+          '[' << obj.inject(""){|acc,o| acc.empty? ? Misc.digest_str(o) : acc << ', ' << Misc.digest_str(o) } << ']'
+        end
       when Hash
         '{' << obj.inject(""){|acc,p| s = Misc.digest_str(p.first) << "=" << Misc.digest_str(p.last); acc.empty? ? s : acc << ', ' << s } << '}'
       when Integer
@@ -46,6 +54,7 @@ module Misc
 
   def self.file_md5(file)
     file = file.find if Path === file
+    file = File.expand_path(file)
     #md5file = file + '.md5'
     Persist.persist("MD5:#{file}", :string) do
       Digest::MD5.file(file).hexdigest

data/lib/scout/misc/format.rb

@@ -38,10 +38,10 @@ module Misc
 
 
   
-  MAX_WIDTH = 100
+  MAX_TTY_LINE_WIDTH = 100
   def self.format_paragraph(text, size = nil, indent = nil, offset = nil)
-    size ||= Log.tty_size || MAX_WIDTH
-    size = MAX_WIDTH if size > MAX_WIDTH
+    size ||= Log.tty_size || MAX_TTY_LINE_WIDTH
+    size = MAX_TTY_LINE_WIDTH if size > MAX_TTY_LINE_WIDTH
     indent ||= 0
     offset ||= 0
 
@@ -75,9 +75,14 @@ module Misc
   end
 
   def self.format_definition_list_item(dt, dd,  indent = nil, size = nil, color = :yellow)
-    size ||= Log.tty_size || MAX_WIDTH
-    size = MAX_WIDTH if size > MAX_WIDTH
-    indent ||= size / 3
+    if size.nil?
+      base_size = MAX_TTY_LINE_WIDTH
+      base_indent = indent || (base_size / 3)
+      size = base_size - base_indent
+    end
+
+    indent ||= base_indent || size / 3
+
     dd = "" if dd.nil?
     dt = Log.color color, dt if color
     dt = dt.to_s  unless dd.empty?
@@ -96,8 +101,8 @@ module Misc
   end
 
   def self.format_definition_list(defs, indent = nil, size = nil, color = :yellow, sep = "\n\n")
-    size ||= Log.tty_size || MAX_WIDTH
     indent ||= 30
+    size ||= (Log.tty_size || MAX_TTY_LINE_WIDTH) - indent
     entries = []
     defs.each do |dt,dd|
       text = format_definition_list_item(dt,dd,indent, size,color)

data/lib/scout/misc/monitor.rb

@@ -2,6 +2,7 @@ module Misc
   def self.pid_alive?(pid)
     !! Process.kill(0, pid) rescue false
   end
+
   def self.benchmark(repeats = 1, message = nil)
     require 'benchmark'
     res = nil
@@ -52,4 +53,14 @@ module Misc
     end
     eend - start
   end
+
+  def self.wait_for_interrupt
+    while true
+      begin
+        sleep 1
+      rescue Interrupt
+        break
+      end
+    end
+  end
 end

data/lib/scout/misc/system.rb

@@ -1,4 +1,18 @@
+require 'sys/proctable'
+
 module Misc
+
+  def self.hostname
+    @@hostname ||= begin
+                     `hostname`.strip
+                   end
+  end
+
+  def self.children(ppid = nil)
+    ppid ||= Process.pid
+    Sys::ProcTable.ps.select{ |pe| pe.ppid == ppid }
+  end
+
   def self.env_add(var, value, sep = ":", prepend = true)
     if ENV[var].nil?
       ENV[var] = value
@@ -22,4 +36,38 @@ module Misc
       ENV[var] = old_value
     end
   end
+  
+  def self.update_git(gem_name = 'scout-gear')
+    gem_name = 'scout-gear' if gem_name.nil?
+    dir = File.join(__dir__, '../../../../', gem_name)
+    return unless Open.exist?(dir)
+    Misc.in_dir dir do
+      begin
+        begin
+          CMD.cmd_log('git pull')
+        rescue
+          raise "Could not update #{gem_name}"
+        end
+
+        begin
+          CMD.cmd_log('git submodule update')
+        rescue
+          raise "Could not update #{gem_name} submodules"
+        end
+
+
+        begin
+          CMD.cmd_log('rake install')
+        rescue
+          raise "Could not install updated #{gem_name}"
+        end
+      rescue
+        Log.warn $!.message
+      end
+    end
+  end
+
+  def self.processors
+    Etc.nprocessors
+  end
 end

data/lib/scout/named_array.rb

@@ -127,4 +127,12 @@ module NamedArray
     end
     source
   end
+
+  def method_missing(name, *args)
+    if identify_name(name)
+      return self[name]
+    else
+      return super(name, *args)
+    end
+  end
 end

data/lib/scout/offsite/ssh.rb

@@ -0,0 +1,171 @@
+require 'net/ssh'
+
+class SSHLine
+  class << self
+    attr_accessor :default_server
+    def default_server
+      @@default_server ||= begin
+                             ENV["SCOUT_OFFSITE"] || ENV["SCOUT_SERVER"] || 'localhost'
+                           end
+    end
+  end
+
+  def initialize(host = :default, user = nil)
+    host = SSHLine.default_server if host.nil? || host == :default
+    @host = host
+    @user = user
+
+    @ssh = Net::SSH.start(@host, @user)
+
+    @ch = @ssh.open_channel do |ch|
+      ch.exec 'bash -l'
+    end
+
+    @ch.on_data do |_,data|
+      if m = data.match(/DONECMD: (\d+)\n/)
+        @exit_status = m[1].to_i
+        @output << data.sub(m[0],'')
+        serve_output 
+      else
+        @output << data
+      end
+    end
+
+    @ch.on_extended_data do |_,c,err|
+      STDERR.write err 
+    end
+  end
+
+
+  def self.reach?(server = SSHLine.default_server)
+    Persist.memory(server, :key => "Reach server") do
+      begin
+        CMD.cmd("ssh #{server} bash -l -c \"scout\"") 
+        true
+      rescue Exception
+        false
+      end
+    end
+  end
+
+  def send_cmd(command)
+    @output = ""
+    @complete_output = false
+    @ch.send_data(command+"\necho DONECMD: $?\n")
+  end
+
+  def serve_output
+    @complete_output = true
+  end
+
+  def run(command)
+    send_cmd(command)
+    @ssh.loop{ ! @complete_output}
+    if @exit_status.to_i == 0
+      return @output
+    else
+      raise SSHProcessFailed.new @host, command
+    end
+  end
+
+  def ruby(script)
+    @output = ""
+    @complete_output = false
+    cmd = "ruby -e \"#{script.gsub('"','\\"')}\"\n"
+    Log.debug "Running ruby on #{@host}:\n#{ script }"
+    @ch.send_data(cmd)
+    @ch.send_data("echo DONECMD: $?\n")
+    @ssh.loop{ !@complete_output }
+    if @exit_status.to_i == 0
+      return @output
+    else
+      raise SSHProcessFailed.new @host, "Ruby script:\n#{script}"
+    end
+  end
+
+  def scout(script)
+    scout_script =<<-EOF
+require 'scout'
+SSHLine.run_local do
+#{script.strip}
+end
+    EOF
+
+    m = ruby(scout_script)
+    Marshal.load m
+  end
+
+  def workflow(workflow, script)
+    preamble =<<-EOF
+wf = Workflow.require_workflow('#{workflow}')
+    EOF
+
+    scout(preamble + "\n" + script)
+  end
+
+  class Mock < SSHLine
+    def initialize
+    end
+
+    def run(command)
+      CMD.cmd(command)
+    end
+
+    def ruby(script)
+      cmd = "ruby -e \"#{script.gsub('"','\\"')}\"\n"
+      CMD.cmd(cmd)
+    end
+  end
+
+  @connections = {}
+  def self.open(host, user = nil)
+    @connections[[host, user]] ||=
+      begin
+        if host == 'localhost'
+          SSHLine::Mock.new
+        else
+          SSHLine.new host, user
+        end
+      end
+  end
+
+  def self.run(server, cmd, options = nil)
+    cmd = cmd * " " if Array === cmd
+    cmd += " " + CMD.process_cmd_options(options) if options
+    open(server).run(cmd)
+  end
+
+  def self.ruby(server, script)
+    open(server).ruby(script)
+  end
+
+  def self.scout(server, script)
+    open(server).scout(script)
+  end
+
+  def self.workflow(server, workflow, script)
+    open(server).workflow(workflow, script)
+  end
+
+  def self.command(server, command, argv = [], options = nil)
+    command = "scout #{command}" unless command && command.include?('scout')
+    argv_str = (argv - ["--"]).collect{|v| '"' + v.to_s + '"' } * " "
+    command = "#{command} #{argv_str}"
+    Log.debug "Offsite #{server} running: #{command}"
+    run(server, command, options)
+  end
+
+  def self.mkdir(server, path)
+    self.run server, "mkdir -p '#{path}'"
+  end
+
+  def self.run_local(&block)
+    res = begin
+            old_stdout = STDOUT.dup; STDOUT.reopen(STDERR)
+            block.call
+          ensure
+            STDOUT.reopen(old_stdout)
+          end
+    puts Marshal.dump(res)
+  end
+end

data/lib/scout/offsite/step.rb

@@ -0,0 +1,83 @@
+require_relative '../workflow/step'
+require_relative 'ssh'
+require_relative 'sync'
+
+module OffsiteStep
+
+  extend MetaExtension
+  extension_attr :server, :workflow_name, :clean_id, :provided_inputs
+
+  def inputs_directory
+    @inputs_directory ||= begin
+                            if provided_inputs && provided_inputs.any?
+                              file = ".scout/tmp/step_inputs/#{workflow}/#{task_name}/#{name}"
+                              TmpFile.with_path do |inputs_dir|
+                                task.save_inputs(inputs_dir, provided_inputs)
+                                SSHLine.rsync(inputs_dir, file, target: server, directory: true)
+                              end
+                              file
+                            end
+                          end
+  end
+
+  def workflow_name
+    @workflow_name || workflow.to_s
+  end
+
+  def offsite_job_ssh(script)
+    parts = []
+    parts << <<~EOF.strip
+wf = Workflow.require_workflow "#{workflow_name}";
+    EOF
+
+    if inputs_directory
+      parts << <<~EOF.strip
+job = wf.job(:#{task_name}, "#{clean_name}", :load_inputs => "#{inputs_directory}");
+      EOF
+    else
+      parts << <<~EOF.strip
+job = wf.job(:#{task_name}, "#{clean_name}");
+      EOF
+    end
+
+    parts << script
+
+
+    SSHLine.scout server, parts * "\n"
+  end
+
+  def offsite_path
+    @path = offsite_job_ssh <<~EOF
+      job.path.identify
+    EOF
+  end
+
+  def info
+    info = @info ||= offsite_job_ssh <<~EOF
+    info = Open.exists?(job.info_file) ? job.info : {}
+    info[:running] = true if job.running?
+    info
+    EOF
+
+    @info = nil unless %w(done aborted error).include?(info[:status].to_s)
+
+    info
+  end
+
+  def done?
+    status == :done
+  end
+
+  def exec
+    bundle_files = offsite_job_ssh <<~EOF
+    job.run
+    job.bundle_files
+    EOF
+    SSHLine.sync(bundle_files, source: server)
+    self.load
+  end
+
+  def run
+    exec
+  end
+end

data/lib/scout/offsite/sync.rb

@@ -0,0 +1,55 @@
+class SSHLine
+  def self.locate(server, paths, map: :user)
+    SSHLine.scout server, <<-EOF
+map = :#{map}
+paths = [#{paths.collect{|p| "'" + p + "'" } * ", " }]
+located = paths.collect{|p| Path.setup(p).find(map) }
+identified = paths.collect{|p| Resource.identify(p) }
+[located, identified]
+    EOF
+  end
+
+  def self.rsync(source_path, target_path, directory: false, source: nil, target: nil, dry_run: false, hard_link: false)
+    rsync_args = "-avztHP --copy-unsafe-links --omit-dir-times "
+
+    rsync_args << "--link-dest '#{source_path}' " if hard_link && ! source
+
+    source_path = source_path + "/" if directory && ! source_path.end_with?("/")
+    target_path = target_path + "/" if directory && ! target_path.end_with?("/")
+    if target
+      SSHLine.mkdir target, File.dirname(target_path)
+    else
+      Open.mkdir(File.dirname(target_path))
+    end
+
+    cmd = 'rsync '
+    cmd << rsync_args
+    cmd << '-nv ' if dry_run
+    cmd << (source ? [source, source_path] * ":" : source_path) << " "
+    cmd << (target ? [target, target_path] * ":" : target_path) << " "
+
+    CMD.cmd_log(cmd, :log => Log::HIGH)
+  end
+
+  def self.sync(paths, source: nil, target: nil, map: :user, **kwargs)
+    source = nil if source == 'localhost'
+    target = nil if target == 'localhost'
+
+    if source
+      source_paths, identified_paths = SSHLine.locate(source, paths)
+    else
+      source_paths = paths.collect{|p| Path === p ? p.find : p }
+      identified_paths = paths.collect{|p| Resource.identify(p) }
+    end
+
+    if target
+      target_paths = SSHLine.locate(target, identified_paths, map: map)
+    else
+      target_paths = identified_paths.collect{|p| p.find(map) }
+    end
+
+    source_paths.zip(target_paths).each do |source_path,target_path|
+      rsync(source_path, target_path, source: source, target: target, **kwargs)
+    end
+  end
+end