scout-gear 8.0.0 → 9.0.0

data/lib/scout/offsite/ssh.rb

@@ -0,0 +1,174 @@
+require 'net/ssh'
+
+class SSHLine
+  class << self
+    attr_accessor :default_server
+    def default_server
+      @@default_server ||= begin
+                             ENV["SCOUT_OFFSITE"] || ENV["SCOUT_SERVER"] || 'localhost'
+                           end
+    end
+  end
+
+  def initialize(host = :default, user = nil)
+    host = SSHLine.default_server if host.nil? || host == :default
+    @host = host
+    @user = user
+
+    @ssh = Net::SSH.start(@host, @user)
+
+    @ch = @ssh.open_channel do |ch|
+      ch.exec 'bash -l'
+    end
+
+    @ch.send_data("[[ -f ~/.scout/environment ]] && source ~/.scout/environment\n")
+    @ch.send_data("[[ -f ~/.rbbt/environment ]] && source ~/.rbbt/environment\n")
+
+    @ch.on_data do |_,data|
+      if m = data.match(/DONECMD: (\d+)\n/)
+        @exit_status = m[1].to_i
+        @output << data.sub(m[0],'')
+        serve_output 
+      else
+        @output << data
+      end
+    end
+
+    @ch.on_extended_data do |_,c,err|
+      STDERR.write err 
+    end
+  end
+
+
+  def self.reach?(server = SSHLine.default_server)
+    Persist.memory(server, :key => "Reach server") do
+      begin
+        CMD.cmd("ssh #{server} bash -l -c \"scout\"") 
+        true
+      rescue Exception
+        false
+      end
+    end
+  end
+
+  def send_cmd(command)
+    @output = ""
+    @complete_output = false
+    @ch.send_data(command+"\necho DONECMD: $?\n")
+  end
+
+  def serve_output
+    @complete_output = true
+  end
+
+  def run(command)
+    send_cmd(command)
+    @ssh.loop{ ! @complete_output}
+    if @exit_status.to_i == 0
+      return @output
+    else
+      raise SSHProcessFailed.new @host, command
+    end
+  end
+
+  def ruby(script)
+    @output = ""
+    @complete_output = false
+    cmd = "ruby -e \"#{script.gsub('"','\\"')}\"\n"
+    Log.debug "Running ruby on #{@host}:\n#{ script }"
+    @ch.send_data(cmd)
+    @ch.send_data("echo DONECMD: $?\n")
+    @ssh.loop{ !@complete_output }
+    if @exit_status.to_i == 0
+      return @output
+    else
+      raise SSHProcessFailed.new @host, "Ruby script:\n#{script}"
+    end
+  end
+
+  def scout(script)
+    scout_script =<<-EOF
+require 'scout'
+SSHLine.run_local do
+#{script.strip}
+end
+    EOF
+
+    m = ruby(scout_script)
+    Marshal.load m
+  end
+
+  def workflow(workflow, script)
+    preamble =<<-EOF
+wf = Workflow.require_workflow('#{workflow}')
+    EOF
+
+    scout(preamble + "\n" + script)
+  end
+
+  class Mock < SSHLine
+    def initialize
+    end
+
+    def run(command)
+      CMD.cmd(command)
+    end
+
+    def ruby(script)
+      cmd = "ruby -e \"#{script.gsub('"','\\"')}\"\n"
+      CMD.cmd(cmd)
+    end
+  end
+
+  @connections = {}
+  def self.open(host, user = nil)
+    @connections[[host, user]] ||=
+      begin
+        if host == 'localhost'
+          SSHLine::Mock.new
+        else
+          SSHLine.new host, user
+        end
+      end
+  end
+
+  def self.run(server, cmd, options = nil)
+    cmd = cmd * " " if Array === cmd
+    cmd += " " + CMD.process_cmd_options(options) if options
+    open(server).run(cmd)
+  end
+
+  def self.ruby(server, script)
+    open(server).ruby(script)
+  end
+
+  def self.scout(server, script)
+    open(server).scout(script)
+  end
+
+  def self.workflow(server, workflow, script)
+    open(server).workflow(workflow, script)
+  end
+
+  def self.command(server, command, argv = [], options = nil)
+    command = "scout #{command}" unless command && command.include?('scout')
+    argv_str = (argv - ["--"]).collect{|v| '"' + v.to_s + '"' } * " "
+    command = "#{command} #{argv_str}"
+    Log.debug "Offsite #{server} running: #{command}"
+    run(server, command, options)
+  end
+
+  def self.mkdir(server, path)
+    self.run server, "mkdir -p '#{path}'"
+  end
+
+  def self.run_local(&block)
+    res = begin
+            old_stdout = STDOUT.dup; STDOUT.reopen(STDERR)
+            block.call
+          ensure
+            STDOUT.reopen(old_stdout)
+          end
+    puts Marshal.dump(res)
+  end
+end

data/lib/scout/offsite/step.rb

@@ -0,0 +1,100 @@
+require_relative '../workflow/step'
+require_relative 'ssh'
+require_relative 'sync'
+
+module OffsiteStep
+
+  extend MetaExtension
+  extension_attr :server, :workflow_name, :clean_id, :slurm
+
+  def inputs_directory
+    @inputs_directory ||= begin
+                            if provided_inputs && provided_inputs.any?
+                              file = ".scout/tmp/step_inputs/#{workflow}/#{task_name}/#{name}"
+                              TmpFile.with_path do |inputs_dir|
+                                save_inputs(inputs_dir)
+                                SSHLine.rsync(inputs_dir, file, target: server, directory: true)
+                              end
+                              file
+                            end
+                          end
+  end
+
+  def workflow_name
+    @workflow_name || workflow.to_s
+  end
+
+  def offsite_job_ssh(script)
+    parts = []
+    parts << <<~EOF.strip
+wf = Workflow.require_workflow "#{workflow_name}";
+    EOF
+
+    if inputs_directory
+      parts << <<~EOF.strip
+job = wf.job(:#{task_name}, "#{clean_name}", :load_inputs => "#{inputs_directory}");
+      EOF
+    else
+      parts << <<~EOF.strip
+job = wf.job(:#{task_name}, "#{clean_name}");
+      EOF
+    end
+
+    parts << script
+
+
+    SSHLine.scout server, parts * "\n"
+  end
+
+  def offsite_path
+    @path = offsite_job_ssh <<~EOF
+      job.path.identify
+    EOF
+  end
+
+  def info
+    info = @info ||= offsite_job_ssh <<~EOF
+    info = Open.exists?(job.info_file) ? job.info : {}
+    info[:running] = true if job.running?
+    info
+    EOF
+
+    @info = nil unless %w(done aborted error).include?(info[:status].to_s)
+
+    info
+  end
+
+  def done?
+    status == :done
+  end
+
+  def orchestrate_slurm
+    bundle_files = offsite_job_ssh <<~EOF
+    require 'rbbt/hpc'
+    HPC::BATCH_MODULE = HPC.batch_system "SLURM"
+    HPC::BATCH_MODULE.orchestrate_job(job, {})
+    job.join
+    job.bundle_files
+    EOF
+    SSHLine.sync(bundle_files, source: server)
+    self.load
+  end
+
+
+  def exec
+    bundle_files = offsite_job_ssh <<~EOF
+    job.run
+    job.bundle_files
+    EOF
+    SSHLine.sync(bundle_files, source: server)
+    self.load
+  end
+
+  def run
+    if slurm
+      orchestrate_slurm
+    else
+      exec
+    end
+  end
+end

data/lib/scout/offsite/sync.rb

@@ -0,0 +1,55 @@
+class SSHLine
+  def self.locate(server, paths, map: :user)
+    SSHLine.scout server, <<-EOF
+map = :#{map}
+paths = [#{paths.collect{|p| "'" + p + "'" } * ", " }]
+located = paths.collect{|p| Path.setup(p).find(map) }
+identified = paths.collect{|p| Resource.identify(p) }
+[located, identified]
+    EOF
+  end
+
+  def self.rsync(source_path, target_path, directory: false, source: nil, target: nil, dry_run: false, hard_link: false)
+    rsync_args = "-avztHP --copy-unsafe-links --omit-dir-times "
+
+    rsync_args << "--link-dest '#{source_path}' " if hard_link && ! source
+
+    source_path = source_path + "/" if directory && ! source_path.end_with?("/")
+    target_path = target_path + "/" if directory && ! target_path.end_with?("/")
+    if target
+      SSHLine.mkdir target, File.dirname(target_path)
+    else
+      Open.mkdir(File.dirname(target_path))
+    end
+
+    cmd = 'rsync '
+    cmd << rsync_args
+    cmd << '-nv ' if dry_run
+    cmd << (source ? [source, source_path] * ":" : source_path) << " "
+    cmd << (target ? [target, target_path] * ":" : target_path) << " "
+
+    CMD.cmd_log(cmd, :log => Log::HIGH)
+  end
+
+  def self.sync(paths, source: nil, target: nil, map: :user, **kwargs)
+    source = nil if source == 'localhost'
+    target = nil if target == 'localhost'
+
+    if source
+      source_paths, identified_paths = SSHLine.locate(source, paths)
+    else
+      source_paths = paths.collect{|p| Path === p ? p.find : p }
+      identified_paths = paths.collect{|p| Resource.identify(p) }
+    end
+
+    if target
+      target_paths = SSHLine.locate(target, identified_paths, map: map)
+    else
+      target_paths = identified_paths.collect{|p| p.find(map) }
+    end
+
+    source_paths.zip(target_paths).each do |source_path,target_path|
+      rsync(source_path, target_path, source: source, target: target, **kwargs)
+    end
+  end
+end

data/lib/scout/offsite.rb

@@ -0,0 +1,3 @@
+require_relative 'offsite/ssh'
+require_relative 'offsite/step'
+require_relative 'offsite/sync'

data/lib/scout/open/lock.rb

@@ -4,27 +4,13 @@ require_relative '../exceptions'
 require_relative 'lock/lockfile'
 
 module Open
-  def self.thread_in_lock(t)
-    t.backtrace.select{|l| l.include?("lockfile") }.any?
-  end
-
-  def self.unlock_thread(t, exception = nil)
-    while t.alive? && t.backtrace.select{|l| l.include?("lockfile") }.any?
-      iii :UNLOCK
-      t.raise(exception)
-      Log.stack t.backtrace
-      locks = t["locks"]
-      return if locks.nil? || locks.empty?
-      locks.each do |lock|
-        lock.unlock if lock.locked?
-        Open.rm(lock.path)
-      end
-      iii Thread.current
-      Thread.list.each{|t| iii [t, t["locks"]]; Log.stack t.backtrace }
-      Log.stack t.backtrace
-    end
+  def self.init_lock
+    Lockfile.refresh = 2 
+    Lockfile.max_age = 30
+    Lockfile.suspend = 4
   end
 
+  self.init_lock 
 
   def self.lock(file, unlock = true, options = {})
     unlock, options = true, unlock if Hash === unlock
@@ -57,9 +43,6 @@ module Open
       raise LockInterrupted
     end
 
-    iii Thread.current["lock_exception"] if Thread.current["lock_exception"]
-    raise Thread.current["lock_exception"] if Thread.current["lock_exception"]
-
     res = nil
 
     begin
@@ -73,8 +56,6 @@ module Open
           if lockfile.locked?
             lockfile.unlock 
           end
-          iii Thread.current["lock_exception"] if Thread.current["lock_exception"]
-          raise Thread.current["lock_exception"] if Thread.current["lock_exception"]
         rescue Exception
           Log.warn "Exception unlocking: #{lockfile.path}"
           Log.exception $!

data/lib/scout/open/remote.rb

@@ -23,7 +23,11 @@ module Open
     m = file.match(/ssh:\/\/([^:]+):(.*)/)
     server = m[1]
     file = m[2]
-    CMD.cmd("ssh '#{server}' cat '#{file}'", :pipe => true, :autojoin => true)
+    if server == 'localhost'
+      Open.open(file)
+    else
+      CMD.cmd("ssh '#{server}' cat '#{file}'", :pipe => true, :autojoin => true)
+    end
   end
 
   def self.wget(url, options = {})
@@ -121,4 +125,11 @@ module Open
     filename = cache_file(url, options)
     Open.open(filename)
   end
+
+  def self.scp(source_file, target_file, target: nil, source: nil)
+    CMD.cmd_log("ssh #{target} mkdir -p #{File.dirname(target_file)}")
+    target_file = [target, target_file] * ":" if target && ! target_file.start_with?(target+":")
+    source_file = [source, source_file] * ":" if source && ! source_file.start_with?(source+":")
+    CMD.cmd_log("scp -r '#{ source_file }' #{target_file}")
+  end
 end