scout-gear 10.7.4 → 10.7.6

data/lib/scout/offsite/ssh.rb

@@ -1,175 +0,0 @@
-require 'net/ssh'
-require_relative 'exceptions'
-
-class SSHLine
-  class << self
-    attr_accessor :default_server
-    def default_server
-      @@default_server ||= begin
-                             ENV["SCOUT_OFFSITE"] || ENV["SCOUT_SERVER"] || 'localhost'
-                           end
-    end
-  end
-
-  def initialize(host = :default, user = nil)
-    host = SSHLine.default_server if host.nil? || host == :default
-    @host = host
-    @user = user
-
-    @ssh = Net::SSH.start(@host, @user)
-
-    @ch = @ssh.open_channel do |ch|
-      ch.exec 'bash -l'
-    end
-
-    @ch.send_data("[[ -f ~/.scout/environment ]] && source ~/.scout/environment\n")
-    @ch.send_data("[[ -f ~/.rbbt/environment ]] && source ~/.rbbt/environment\n")
-
-    @ch.on_data do |_,data|
-      if m = data.match(/DONECMD: (\d+)\n/)
-        @exit_status = m[1].to_i
-        @output << data.sub(m[0],'')
-        serve_output 
-      else
-        @output << data
-      end
-    end
-
-    @ch.on_extended_data do |_,c,err|
-      STDERR.write err 
-    end
-  end
-
-
-  def self.reach?(server = SSHLine.default_server)
-    Persist.memory(server, :key => "Reach server") do
-      begin
-        CMD.cmd("ssh #{server} bash -l -c \"scout\"") 
-        true
-      rescue Exception
-        false
-      end
-    end
-  end
-
-  def send_cmd(command)
-    @output = ""
-    @complete_output = false
-    @ch.send_data(command+"\necho DONECMD: $?\n")
-  end
-
-  def serve_output
-    @complete_output = true
-  end
-
-  def run(command)
-    send_cmd(command)
-    @ssh.loop{ ! @complete_output}
-    if @exit_status.to_i == 0
-      return @output
-    else
-      raise SSHProcessFailed.new @host, command
-    end
-  end
-
-  def ruby(script)
-    @output = ""
-    @complete_output = false
-    cmd = "ruby -e \"#{script.gsub('"','\\"')}\"\n"
-    Log.debug "Running ruby on #{@host}:\n#{ script }"
-    @ch.send_data(cmd)
-    @ch.send_data("echo DONECMD: $?\n")
-    @ssh.loop{ !@complete_output }
-    if @exit_status.to_i == 0
-      return @output
-    else
-      raise SSHProcessFailed.new @host, "Ruby script:\n#{script}"
-    end
-  end
-
-  def scout(script)
-    scout_script =<<-EOF
-require 'scout'
-SSHLine.run_local do
-#{script.strip}
-end
-    EOF
-
-    m = ruby(scout_script)
-    Marshal.load m
-  end
-
-  def workflow(workflow, script)
-    preamble =<<-EOF
-wf = Workflow.require_workflow('#{workflow}')
-    EOF
-
-    scout(preamble + "\n" + script)
-  end
-
-  class Mock < SSHLine
-    def initialize
-    end
-
-    def run(command)
-      CMD.cmd(command)
-    end
-
-    def ruby(script)
-      cmd = "ruby -e \"#{script.gsub('"','\\"')}\"\n"
-      CMD.cmd(cmd)
-    end
-  end
-
-  @connections = {}
-  def self.open(host, user = nil)
-    @connections[[host, user]] ||=
-      begin
-        if host == 'localhost'
-          SSHLine::Mock.new
-        else
-          SSHLine.new host, user
-        end
-      end
-  end
-
-  def self.run(server, cmd, options = nil)
-    cmd = cmd * " " if Array === cmd
-    cmd += " " + CMD.process_cmd_options(options) if options
-    open(server).run(cmd)
-  end
-
-  def self.ruby(server, script)
-    open(server).ruby(script)
-  end
-
-  def self.scout(server, script)
-    open(server).scout(script)
-  end
-
-  def self.workflow(server, workflow, script)
-    open(server).workflow(workflow, script)
-  end
-
-  def self.command(server, command, argv = [], options = nil)
-    command = "scout #{command}" unless command && command.include?('scout')
-    argv_str = (argv - ["--"]).collect{|v| '"' + v.to_s + '"' } * " "
-    command = "#{command} #{argv_str}"
-    Log.debug "Offsite #{server} running: #{command}"
-    run(server, command, options)
-  end
-
-  def self.mkdir(server, path)
-    self.run server, "mkdir -p '#{path}'"
-  end
-
-  def self.run_local(&block)
-    res = begin
-            old_stdout = STDOUT.dup; STDOUT.reopen(STDERR)
-            block.call
-          ensure
-            STDOUT.reopen(old_stdout)
-          end
-    puts Marshal.dump(res)
-  end
-end

data/lib/scout/offsite/step.rb

@@ -1,100 +0,0 @@
-require_relative '../workflow/step'
-require_relative 'ssh'
-require_relative 'sync'
-
-module OffsiteStep
-
-  extend Annotation
-  annotation :server, :workflow_name, :clean_id, :slurm
-
-  def inputs_directory
-    @inputs_directory ||= begin
-                            if provided_inputs && provided_inputs.any?
-                              file = ".scout/tmp/step_inputs/#{workflow}/#{task_name}/#{name}"
-                              TmpFile.with_path do |inputs_dir|
-                                save_inputs(inputs_dir)
-                                SSHLine.rsync(inputs_dir, file, target: server, directory: true)
-                              end
-                              file
-                            end
-                          end
-  end
-
-  def workflow_name
-    @workflow_name || workflow.to_s
-  end
-
-  def offsite_job_ssh(script)
-    parts = []
-    parts << <<~EOF.strip
-wf = Workflow.require_workflow "#{workflow_name}";
-    EOF
-
-    if inputs_directory
-      parts << <<~EOF.strip
-job = wf.job(:#{task_name}, "#{clean_name}", :load_inputs => "#{inputs_directory}");
-      EOF
-    else
-      parts << <<~EOF.strip
-job = wf.job(:#{task_name}, "#{clean_name}");
-      EOF
-    end
-
-    parts << script
-
-
-    SSHLine.scout server, parts * "\n"
-  end
-
-  def offsite_path
-    @path = offsite_job_ssh <<~EOF
-      job.path.identify
-    EOF
-  end
-
-  def info
-    info = @info ||= offsite_job_ssh <<~EOF
-    info = Open.exists?(job.info_file) ? job.info : {}
-    info[:running] = true if job.running?
-    info
-    EOF
-
-    @info = nil unless %w(done aborted error).include?(info[:status].to_s)
-
-    info
-  end
-
-  def done?
-    status == :done
-  end
-
-  def orchestrate_slurm
-    bundle_files = offsite_job_ssh <<~EOF
-    require 'rbbt/hpc'
-    HPC::BATCH_MODULE = HPC.batch_system "SLURM"
-    HPC::BATCH_MODULE.orchestrate_job(job, {})
-    job.join
-    job.bundle_files
-    EOF
-    SSHLine.sync(bundle_files, source: server)
-    self.load
-  end
-
-
-  def exec
-    bundle_files = offsite_job_ssh <<~EOF
-    job.run
-    job.bundle_files
-    EOF
-    SSHLine.sync(bundle_files, source: server)
-    self.load
-  end
-
-  def run
-    if slurm
-      orchestrate_slurm
-    else
-      exec
-    end
-  end
-end

data/lib/scout/offsite/sync.rb

@@ -1,55 +0,0 @@
-class SSHLine
-  def self.locate(server, paths, map: :user)
-    SSHLine.scout server, <<-EOF
-map = :#{map}
-paths = [#{paths.collect{|p| "'" + p + "'" } * ", " }]
-located = paths.collect{|p| Path.setup(p).find(map) }
-identified = paths.collect{|p| Resource.identify(p) }
-[located, identified]
-    EOF
-  end
-
-  def self.rsync(source_path, target_path, directory: false, source: nil, target: nil, dry_run: false, hard_link: false)
-    rsync_args = "-avztHP --copy-unsafe-links --omit-dir-times "
-
-    rsync_args << "--link-dest '#{source_path}' " if hard_link && ! source
-
-    source_path = source_path + "/" if directory && ! source_path.end_with?("/")
-    target_path = target_path + "/" if directory && ! target_path.end_with?("/")
-    if target
-      SSHLine.mkdir target, File.dirname(target_path)
-    else
-      Open.mkdir(File.dirname(target_path))
-    end
-
-    cmd = 'rsync '
-    cmd << rsync_args
-    cmd << '-nv ' if dry_run
-    cmd << (source ? [source, source_path] * ":" : source_path) << " "
-    cmd << (target ? [target, target_path] * ":" : target_path) << " "
-
-    CMD.cmd_log(cmd, :log => Log::HIGH)
-  end
-
-  def self.sync(paths, source: nil, target: nil, map: :user, **kwargs)
-    source = nil if source == 'localhost'
-    target = nil if target == 'localhost'
-
-    if source
-      source_paths, identified_paths = SSHLine.locate(source, paths)
-    else
-      source_paths = paths.collect{|p| Path === p ? p.find : p }
-      identified_paths = paths.collect{|p| Resource.identify(p) }
-    end
-
-    if target
-      target_paths = SSHLine.locate(target, identified_paths, map: map)
-    else
-      target_paths = identified_paths.collect{|p| p.find(map) }
-    end
-
-    source_paths.zip(target_paths).each do |source_path,target_path|
-      rsync(source_path, target_path, source: source, target: target, **kwargs)
-    end
-  end
-end

data/lib/scout/offsite.rb

@@ -1,3 +0,0 @@
-require_relative 'offsite/ssh'
-require_relative 'offsite/step'
-require_relative 'offsite/sync'

data/scout_commands/offsite

@@ -1,30 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'scout'
-require 'scout/offsite'
-
-$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
-
-options = SOPT.setup <<EOF
-
-Description of the tool
-
-$ #{$0} [<options>] <where> <command> ... -- [<protected options>]
-
-Run a command offsite. Use -- to make sure options reach the offsite
-
--h--help Print this help
-EOF
-if options[:help]
-  if defined? scout_usage
-    scout_usage 
-  else
-    puts SOPT.doc
-  end
-  exit 0
-end
-
-where = ARGV.shift
-command = ARGV.shift
-
-puts SSHLine.command(where, command, ARGV)

data/test/scout/offsite/test_ssh.rb

@@ -1,15 +0,0 @@
-require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
-require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
-
-class TestSSH < Test::Unit::TestCase
-  def test_marshal
-    return unless SSHLine.reach?
-
-    assert TrueClass === SSHLine.scout(:default, 'true')
-  end
-
-  def test_localhost
-    assert SSHLine.scout('localhost', 'true')
-  end
-end
-

data/test/scout/offsite/test_step.rb

@@ -1,32 +0,0 @@
-require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
-require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
-
-class TestOffsiteStep < Test::Unit::TestCase
-  def test_offsite_task
-    workflow_code =<<-EOF
-module TestWF
-  extend Workflow
-
-  input :string, :string, "String", "string"
-  task :string => :string do |string| string end
-end
-
-TestWF.directory = Path.setup("#{tmpdir.offsite.TestWF}")
-    EOF
-
-    TmpFile.with_file workflow_code, :extension => 'rb' do |wffile|
-      wf = Workflow.require_workflow wffile
-
-      job = wf.job(:string)
-
-      off = OffsiteStep.setup job, server: 'localhost', workflow_name: wffile
-
-      refute off.done?
-      assert_equal 'string', off.run
-
-      assert off.done?
-      assert_equal 'string', off.run
-    end
-  end
-end
-

data/test/scout/offsite/test_sync.rb

@@ -1,36 +0,0 @@
-require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
-require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
-
-class TestSync < Test::Unit::TestCase
-  def test_sync
-    TmpFile.with_path do |tmpdir|
-      tmpdir = Scout.tmp.tmpdir_sync
-      tmpdir.dir1.foo.write("FOO")
-      tmpdir.dir1.bar.write("BAR")
-
-      TmpFile.with_path do |tmpdir2|
-        Misc.in_dir tmpdir2 do
-          SSHLine.sync([tmpdir.dir1], map: :current)
-
-          assert tmpdir2.glob("**/*").select{|f| f.include?('foo') }.any?
-        end
-      end
-    end
-  end
-
-  def test_sync_dir_map
-    TmpFile.with_path do |tmpdir|
-      tmpdir = Scout.tmp.tmpdir_sync
-      tmpdir.dir1.foo.write("FOO")
-      tmpdir.dir1.bar.write("BAR")
-
-      TmpFile.with_path do |tmpdir2|
-        SSHLine.sync([tmpdir.dir1], map: tmpdir2)
-        Misc.in_dir tmpdir2 do
-          assert tmpdir2.glob("**/*").select{|f| f.include?('foo') }.any?
-        end
-      end
-    end
-  end
-end
-