scout-camp 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 6f890271ab935a22aef0383dbb515abf6198f4e91d965055655c862066a09f5a
+  data.tar.gz: 04bf98b46eefc8e63a7b1ef158797f01ecb8aa3db24a92d42721eec2df2cefa5
+SHA512:
+  metadata.gz: 6e8bb582fc33d1facaccb53f6e3d762f77a7dc41366a46bb032f65edb73bd976f9c0edf69fee313764e295417ad324c136366c49e5bcee0d0f927f4de7397dd2
+  data.tar.gz: 7245f12146cadce9940e2d59d0cef3b6843ef0139cada356995236c39bf75b104888f9192fca247c98775cd2b3235e51bc7c661f502f9914fdb7c0998752a6fd

data/.vimproject

@@ -0,0 +1,81 @@
+scout-camp=/$PWD filter="*" {
+ bin=bin {
+  scout-camp
+ }
+ lib=lib {
+  scout-camp.rb
+  scout=scout {
+   aws=aws{
+    s3.rb
+   }
+   terraform_dsl.rb
+   terraform_dsl=terraform_dsl {
+    deployment.rb
+    util.rb
+   }
+   offsite.rb
+   offsite=offsite {
+    exceptions.rb
+    ssh.rb
+    step.rb
+    sync.rb
+   }
+  }
+ }
+ scout_commands=scout_commands {
+  offsite
+  terraform=terraform{
+   list
+   add
+   status
+   apply
+   destroy
+   remove
+  }
+ }
+ share=share {
+  terraform=terraform {
+   aws=aws {
+    bucket=bucket {
+     main.tf
+     output.tf
+     variables.tf
+    }
+    cluster=cluster {
+     main.tf
+     output.tf
+     variables.tf
+    }
+    host=host {
+     locals.tf
+     main.tf
+     output.tf
+     variables.tf
+    }
+    lambda=lambda {
+     main.tf
+     variables.tf
+    }
+    provider=provider {
+     data.tf
+     output.tf
+    }
+   }
+  }
+ }
+ test=test {
+  test_helper.rb
+  scout=scout {
+   test_terraform_dsl.rb
+   offsite=offsite {
+    test_ssh.rb
+    test_step.rb
+    test_sync.rb
+   }
+  }
+ }
+ LICENSE
+ README.md
+ Rakefile
+ VERSION
+}

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2025 Miguel Vazquez
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,26 @@
+scout-camp===================
+
+Description goes here.
+
+Contributing to scout-camp
+------------------------------------------
+
+-   Check out the latest master to make sure the feature hasn't been
+    implemented or the bug hasn't been fixed yet.
+-   Check out the issue tracker to make sure someone already hasn't
+    requested it and/or contributed it.
+-   Fork the project.
+-   Start a feature/bugfix branch.
+-   Commit and push until you are happy with your contribution.
+-   Make sure to add tests for it. This is important so I don't break it
+    in a future version unintentionally.
+-   Please try not to mess with the Rakefile, version, or history. If
+    you want to have your own version, or is otherwise necessary, that
+    is fine, but please isolate to its own commit so I can cherry-pick
+    around it.
+
+Copyright
+---------
+
+Copyright (c) 2025 Miguel Vazquez. See
+LICENSE.txt for further details.

data/Rakefile

@@ -0,0 +1,70 @@
+# encoding: utf-8
+
+ENV["BRANCH"] = 'main'
+
+require 'rubygems'
+require 'rake'
+require 'juwelier'
+Juwelier::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://guides.rubygems.org/specification-reference/ for more options
+  gem.name = "scout-camp"
+  gem.homepage = "http://github.com/mikisvaz/scout-camp"
+  gem.license = "MIT"
+  gem.summary = %Q{Deploy you scouts}
+  gem.description = %Q{Functionalities to deploy and use scouts in remote servers like AWS}
+  gem.email = "mikisvaz@gmail.com"
+  gem.authors = ["Miguel Vazquez"]
+
+  # Include your dependencies below. Runtime dependencies are required when using your gem,
+  # and development dependencies are only needed for development (ie running rake tasks, tests, etc)
+  #  gem.add_runtime_dependency 'jabber4r', '> 0.1'
+  #  gem.add_development_dependency 'rspec', '> 1.2.3'
+  #gem.add_development_dependency "shoulda", ">= 0"
+  # Include your dependencies below. Runtime dependencies are required when using your gem,
+  # and development dependencies are only needed for development (ie running rake tasks, tests, etc)
+  #  gem.add_runtime_dependency 'jabber4r', '> 0.1'
+  #  gem.add_development_dependency 'rspec', '> 1.2.3'
+  #gem.add_development_dependency "rdoc", "~> 3.12"
+  # Include your dependencies below. Runtime dependencies are required when using your gem,
+  # and development dependencies are only needed for development (ie running rake tasks, tests, etc)
+  #  gem.add_runtime_dependency 'jabber4r', '> 0.1'
+  #  gem.add_development_dependency 'rspec', '> 1.2.3'
+  #gem.add_development_dependency "bundler", "~> 1.0"
+  # Include your dependencies below. Runtime dependencies are required when using your gem,
+  # and development dependencies are only needed for development (ie running rake tasks, tests, etc)
+  #  gem.add_runtime_dependency 'jabber4r', '> 0.1'
+  #  gem.add_development_dependency 'rspec', '> 1.2.3'
+  #gem.add_development_dependency "juwelier", "~> 2.1.0"
+  # Include your dependencies below. Runtime dependencies are required when using your gem,
+  # and development dependencies are only needed for development (ie running rake tasks, tests, etc)
+  #  gem.add_runtime_dependency 'jabber4r', '> 0.1'
+  #  gem.add_development_dependency 'rspec', '> 1.2.3'
+  #gem.add_development_dependency "simplecov", ">= 0"
+
+  gem.add_runtime_dependency 'scout-essentials', '>= 0'
+end
+Juwelier::RubygemsDotOrgTasks.new
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+desc "Code coverage detail"
+task :simplecov do
+  ENV['COVERAGE'] = "true"
+  Rake::Task['test'].execute
+end
+
+task :default => :test
+
+require 'rdoc/task'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "scout-camp #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.1.1

data/bin/scout-camp

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'scout-camp'
+
+load Scout.bin.scout.find

data/lib/scout/aws/s3.rb

@@ -0,0 +1,157 @@
+require 'aws-sdk-s3'
+require 'uri'
+
+require 'scout/path'
+require 'scout/open'
+require 'scout/misc/hook'
+
+module Open
+  module S3
+    extend Hook
+
+    def self.is_s3?(uri)
+      uri.start_with? 's3://'
+    end
+
+    def self.claim(uri, ...)
+      is_s3? uri
+    end
+
+    def self.parse_s3_uri(uri)
+      uri = uri.sub(%r{^s3://}, '')
+      bucket, *key_parts = uri.split('/', -1)
+      key = key_parts.join('/').sub(%r{^/}, '')
+      [bucket, key]
+    end
+
+    def self.get_stream(uri, *args)
+      bucket, key = parse_s3_uri(uri)
+      return nil if key.empty?
+
+      Open.open_pipe do |sin|
+        s3 = Aws::S3::Client.new
+        s3.get_object(bucket: bucket, key: key) do |block|
+          sin.write block
+        end
+      end
+    rescue Aws::S3::Errors::NoSuchKey
+      nil
+    end
+
+    def self.write(uri, content = nil, &block)
+      bucket, key = parse_s3_uri(uri)
+      s3 = Aws::S3::Client.new
+      content = Open.open_pipe(&block).read if block_given?
+      s3.put_object(bucket: bucket, key: key, body: content)
+    end
+
+    def self.glob(uri, pattern="**/*")
+      bucket, prefix = parse_s3_uri(uri)
+      s3 = Aws::S3::Client.new
+      matches = []
+      continuation_token = nil
+
+      loop do
+        resp = s3.list_objects_v2(
+          bucket: bucket,
+          prefix: prefix,
+          continuation_token: continuation_token
+        )
+
+        resp.contents.each do |object|
+          key = object.key
+          remaining = key[prefix.length..-1] || ''
+          remaining = remaining.sub(%r{^/}, '') if prefix.empty? # Handle root-level keys with leading slash
+
+          if File.fnmatch?(pattern, remaining, File::FNM_PATHNAME)
+            matches << "s3://#{bucket}/#{key}"
+          end
+        end
+
+        continuation_token = resp.next_continuation_token
+        break unless continuation_token
+      end
+
+      matches
+    end
+
+    def self.rm(uri)
+      bucket, key = parse_s3_uri(uri)
+      return false if key.empty? # Prevent accidental bucket deletion attempts
+
+      s3 = Aws::S3::Client.new
+      s3.delete_object(bucket: bucket, key: key)
+      true
+    rescue Aws::S3::Errors::NoSuchKey, Aws::S3::Errors::NoSuchBucket
+      false
+    end
+
+    def self.rm_rf(uri)
+      bucket, prefix = parse_s3_uri(uri)
+      s3 = Aws::S3::Client.new
+      continuation_token = nil
+
+      loop do
+        resp = s3.list_objects_v2(
+          bucket: bucket,
+          prefix: prefix,
+          continuation_token: continuation_token
+        )
+
+        if resp.contents.any?
+          # Delete objects in batches of 1000 (S3 limit per request)
+          objects = resp.contents.map { |obj| { key: obj.key } }
+          s3.delete_objects(
+            bucket: bucket,
+            delete: { objects: objects, quiet: true }
+          )
+        end
+
+        continuation_token = resp.next_continuation_token
+        break unless continuation_token
+      end
+
+      true
+    rescue Aws::S3::Errors::NoSuchBucket
+      false
+    end
+
+    def self.exists?(uri)
+      bucket, key = parse_s3_uri(uri)
+      return false if key.empty? # Can't check existence of bucket this way
+
+      s3 = Aws::S3::Client.new
+      s3.head_object(bucket: bucket, key: key)
+      true
+    rescue Aws::S3::Errors::NotFound, Aws::S3::Errors::NoSuchBucket
+      false
+    end
+
+    self.singleton_class.alias_method :exist?, :exists?
+  
+  end
+end
+
+module Path
+  extend Hook
+
+  module S3
+    def located?
+      Open::S3.is_s3?(self) || orig_located?
+    end
+
+    def glob(*args)
+      if Open::S3.is_s3?(self)
+        Open::S3.glob(self, *args)
+      else
+        orig_glob(*args)
+      end
+    end
+  end
+end
+
+Hook.apply(Open::S3, Open)
+Hook.apply(Path::S3, Path)
+
+
+#$ ask -t code --file /home/miki/git/scout-camp/lib/scout/aws/s3.rb extend this file [[...]] to include a function called self.exists? that determines if a uri exists {{{

data/lib/scout/offsite/exceptions.rb

@@ -0,0 +1,9 @@
+class SSHProcessFailed < StandardError
+  attr_accessor :host, :cmd
+  def initialize(host, cmd)
+    @host = host
+    @cmd = cmd
+    message = "SSH server #{host} failed cmd '#{cmd}'" 
+    super(message)
+  end
+end

data/lib/scout/offsite/ssh.rb

@@ -0,0 +1,175 @@
+require 'net/ssh'
+require_relative 'exceptions'
+
+class SSHLine
+  class << self
+    attr_accessor :default_server
+    def default_server
+      @@default_server ||= begin
+                             ENV["SCOUT_OFFSITE"] || ENV["SCOUT_SERVER"] || 'localhost'
+                           end
+    end
+  end
+
+  def initialize(host = :default, user = nil)
+    host = SSHLine.default_server if host.nil? || host == :default
+    @host = host
+    @user = user
+
+    @ssh = Net::SSH.start(@host, @user)
+
+    @ch = @ssh.open_channel do |ch|
+      ch.exec 'bash -l'
+    end
+
+    @ch.send_data("[[ -f ~/.scout/environment ]] && source ~/.scout/environment\n")
+    @ch.send_data("[[ -f ~/.rbbt/environment ]] && source ~/.rbbt/environment\n")
+
+    @ch.on_data do |_,data|
+      if m = data.match(/DONECMD: (\d+)\n/)
+        @exit_status = m[1].to_i
+        @output << data.sub(m[0],'')
+        serve_output 
+      else
+        @output << data
+      end
+    end
+
+    @ch.on_extended_data do |_,c,err|
+      STDERR.write err 
+    end
+  end
+
+
+  def self.reach?(server = SSHLine.default_server)
+    Persist.memory(server, :key => "Reach server") do
+      begin
+        CMD.cmd("ssh #{server} bash -l -c \"scout\"") 
+        true
+      rescue Exception
+        false
+      end
+    end
+  end
+
+  def send_cmd(command)
+    @output = ""
+    @complete_output = false
+    @ch.send_data(command+"\necho DONECMD: $?\n")
+  end
+
+  def serve_output
+    @complete_output = true
+  end
+
+  def run(command)
+    send_cmd(command)
+    @ssh.loop{ ! @complete_output}
+    if @exit_status.to_i == 0
+      return @output
+    else
+      raise SSHProcessFailed.new @host, command
+    end
+  end
+
+  def ruby(script)
+    @output = ""
+    @complete_output = false
+    cmd = "ruby -e \"#{script.gsub('"','\\"')}\"\n"
+    Log.debug "Running ruby on #{@host}:\n#{ script }"
+    @ch.send_data(cmd)
+    @ch.send_data("echo DONECMD: $?\n")
+    @ssh.loop{ !@complete_output }
+    if @exit_status.to_i == 0
+      return @output
+    else
+      raise SSHProcessFailed.new @host, "Ruby script:\n#{script}"
+    end
+  end
+
+  def scout(script)
+    scout_script =<<-EOF
+require 'scout'
+SSHLine.run_local do
+#{script.strip}
+end
+    EOF
+
+    m = ruby(scout_script)
+    Marshal.load m
+  end
+
+  def workflow(workflow, script)
+    preamble =<<-EOF
+wf = Workflow.require_workflow('#{workflow}')
+    EOF
+
+    scout(preamble + "\n" + script)
+  end
+
+  class Mock < SSHLine
+    def initialize
+    end
+
+    def run(command)
+      CMD.cmd(command)
+    end
+
+    def ruby(script)
+      cmd = "ruby -e \"#{script.gsub('"','\\"')}\"\n"
+      CMD.cmd(cmd)
+    end
+  end
+
+  @connections = {}
+  def self.open(host, user = nil)
+    @connections[[host, user]] ||=
+      begin
+        if host == 'localhost'
+          SSHLine::Mock.new
+        else
+          SSHLine.new host, user
+        end
+      end
+  end
+
+  def self.run(server, cmd, options = nil)
+    cmd = cmd * " " if Array === cmd
+    cmd += " " + CMD.process_cmd_options(options) if options
+    open(server).run(cmd)
+  end
+
+  def self.ruby(server, script)
+    open(server).ruby(script)
+  end
+
+  def self.scout(server, script)
+    open(server).scout(script)
+  end
+
+  def self.workflow(server, workflow, script)
+    open(server).workflow(workflow, script)
+  end
+
+  def self.command(server, command, argv = [], options = nil)
+    command = "scout #{command}" unless command && command.include?('scout')
+    argv_str = (argv - ["--"]).collect{|v| '"' + v.to_s + '"' } * " "
+    command = "#{command} #{argv_str}"
+    Log.debug "Offsite #{server} running: #{command}"
+    run(server, command, options)
+  end
+
+  def self.mkdir(server, path)
+    self.run server, "mkdir -p '#{path}'"
+  end
+
+  def self.run_local(&block)
+    res = begin
+            old_stdout = STDOUT.dup; STDOUT.reopen(STDERR)
+            block.call
+          ensure
+            STDOUT.reopen(old_stdout)
+          end
+    puts Marshal.dump(res)
+  end
+end

data/lib/scout/offsite/step.rb

@@ -0,0 +1,100 @@
+require_relative 'ssh'
+require_relative 'sync'
+require 'scout/workflow/step'
+
+module OffsiteStep
+
+  extend Annotation
+  annotation :server, :workflow_name, :clean_id, :slurm
+
+  def inputs_directory
+    @inputs_directory ||= begin
+                            if provided_inputs && provided_inputs.any?
+                              file = ".scout/tmp/step_inputs/#{workflow}/#{task_name}/#{name}"
+                              TmpFile.with_path do |inputs_dir|
+                                save_inputs(inputs_dir)
+                                SSHLine.rsync(inputs_dir, file, target: server, directory: true)
+                              end
+                              file
+                            end
+                          end
+  end
+
+  def workflow_name
+    @workflow_name || workflow.to_s
+  end
+
+  def offsite_job_ssh(script)
+    parts = []
+    parts << <<~EOF.strip
+wf = Workflow.require_workflow "#{workflow_name}";
+    EOF
+
+    if inputs_directory
+      parts << <<~EOF.strip
+job = wf.job(:#{task_name}, "#{clean_name}", :load_inputs => "#{inputs_directory}");
+      EOF
+    else
+      parts << <<~EOF.strip
+job = wf.job(:#{task_name}, "#{clean_name}");
+      EOF
+    end
+
+    parts << script
+
+
+    SSHLine.scout server, parts * "\n"
+  end
+
+  def offsite_path
+    @path = offsite_job_ssh <<~EOF
+      job.path.identify
+    EOF
+  end
+
+  def info
+    info = @info ||= offsite_job_ssh <<~EOF
+    info = Open.exists?(job.info_file) ? job.info : {}
+    info[:running] = true if job.running?
+    info
+    EOF
+
+    @info = nil unless %w(done aborted error).include?(info[:status].to_s)
+
+    info
+  end
+
+  def done?
+    status == :done
+  end
+
+  def orchestrate_slurm
+    bundle_files = offsite_job_ssh <<~EOF
+    require 'rbbt/hpc'
+    HPC::BATCH_MODULE = HPC.batch_system "SLURM"
+    HPC::BATCH_MODULE.orchestrate_job(job, {})
+    job.join
+    job.bundle_files
+    EOF
+    SSHLine.sync(bundle_files, source: server)
+    self.load
+  end
+
+
+  def exec
+    bundle_files = offsite_job_ssh <<~EOF
+    job.run
+    job.bundle_files
+    EOF
+    SSHLine.sync(bundle_files, source: server)
+    self.load
+  end
+
+  def run
+    if slurm
+      orchestrate_slurm
+    else
+      exec
+    end
+  end
+end