scout-camp 0.1.4 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bc2855face5f3cb3ced365171398295b656f378fc681da6f0cd530c2c2977ea6
-  data.tar.gz: baacce8525b32344f5391d9fa61b62dbb639a91bbddb2ca392a8f7ac8ce762f3
+  metadata.gz: dd5f35a6409cd8556f8fb6cb8641c28c36fb3b0dcc9110eeef8c8d9a5ea42594
+  data.tar.gz: 7bb5e7f5b07ac7d9ca601909bc3c5632d6f9d1630020e308d901d8b3a771d0f5
 SHA512:
-  metadata.gz: c4e5de039eaba61b854f1b0af3a41172d95bbd872f64750d8bd9775b6321bcc0cbe174adbdf684a14ff836195b6cc840be06d97ddd4803ceb2b56cb032302d91
-  data.tar.gz: ca4e6b9e69160ba22e57b3b2cadc9a669f628fdf47b20ea696cf2678b998f686bb6eebc6abcc9aba03369356a5257175b701f080dafd9bb464b3f71cbcceb779
+  metadata.gz: e460301e94edf73163f57309498db7d04b38b202c0437938dc0bf10cb0a31e1683f8d908309b0876fe106e781f2b69e24563656562eb57e0a759116bfb38f904
+  data.tar.gz: 6a78670a91b259399cad19a95411c214f3382173c37bb2611f4b13deb8a32a1626d0f2dfb793e8462253e6322156b104816a0bc5f3f37ce9d0cf3017bf735adf

data/.vimproject

@@ -26,12 +26,17 @@ scout-camp=/$PWD filter="*" {
   offsite
   terraform=terraform{
    list
+   add=add{
+    lambda
+    relay
+   }
    add
    status
    apply
    plan
    destroy
    remove
+   lambda_task
   }
  }
  share=share {
@@ -66,11 +71,36 @@ scout-camp=/$PWD filter="*" {
      output.tf
      variables.tf
     }
+    fargate=fargate{
+     locals.tf
+     main.tf
+     variables.tf
+    }
     provider=provider {
      data.tf
      output.tf
     }
    }
+   ssh=ssh{
+    cmd=cmd{
+     main.tf
+     variables.tf
+    }
+    ollama=ollama{
+     main.tf
+     variables.tf
+    }
+    vllm=vllm{
+     main.tf
+     variables.tf
+    }
+    relay=relay{
+
+    }
+   }
+  }
+  aws=aws{
+   lambda_function.rb
   }
  }
  test=test {

data/VERSION

@@ -1 +1 @@
-0.1.4
+0.1.6

data/lib/scout/aws/s3.rb

@@ -10,7 +10,11 @@ module Open
     extend Hook
 
     def self.lock(*args, &block)
-      yield nil
+      begin
+        yield nil
+      rescue KeepLocked
+        $!.payload
+      end
     end
 
     def self.is_s3?(uri)
@@ -51,6 +55,7 @@ module Open
       bucket, key = parse_s3_uri(uri)
       s3 = Aws::S3::Client.new
       content = Open.open_pipe(&block).read if block_given?
+      content = content.read if IO === content
       s3.put_object(bucket: bucket, key: key, body: content)
     end
 
@@ -69,8 +74,13 @@ module Open
 
         resp.contents.each do |object|
           key = object.key
-          remaining = key[prefix.length..-1] || ''
-          remaining = remaining.sub(%r{^/}, '') if prefix.empty? # Handle root-level keys with leading slash
+
+          if prefix.empty?
+            remaining = key.sub(%r{^/}, '')
+          else
+            remaining = key[prefix.length..-1] || ''
+            remaining = remaining.sub(%r{^/}, '')
+          end
 
           if File.fnmatch?(pattern, remaining, File::FNM_PATHNAME)
             matches << "s3://#{bucket}/#{key}"
@@ -125,6 +135,18 @@ module Open
       false
     end
 
+    def self.cp(source, target)
+      source_bucket, source_key = parse_s3_uri(source)
+      target_bucket, target_key = parse_s3_uri(target)
+
+      s3 = Aws::S3::Client.new
+      s3.copy_object({
+        copy_source: "#{source_bucket}/#{source_key}",
+        bucket: target_bucket,
+        key: target_key
+      })
+    end
+
     def self.exists?(uri)
       bucket, key = parse_s3_uri(uri)
       return false if key.empty? # Can't check existence of bucket this way
@@ -137,12 +159,26 @@ module Open
     end
 
     self.singleton_class.alias_method :exist?, :exists?
-  
 
     def self.sensible_write(path, content = nil, options = {}, &block)
+      content = content.to_s if content.respond_to?(:write_file)
       Open::S3.write(path, content)
     end
 
+    def self.mkdir(path)
+    end
+
+    def self.link(source, target, options = {})
+      cp(source, target)
+    end
+
+    def self.ln(source, target, options = {})
+      cp(source, target)
+    end
+
+    def self.ln_s(source, target, options = {})
+      cp(source, target)
+    end
   end
 end
 
@@ -155,8 +191,8 @@ module Path
     end
 
     def glob(*args)
-      if Open::S3.is_s3?(self)
-        Open::S3.glob(self, *args)
+      if Open::S3.is_s3?(self.find)
+        Open::S3.glob(self.find, *args)
       else
         orig_glob(*args)
       end

data/scout-camp.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-camp 0.1.4 ruby lib
+# stub: scout-camp 0.1.6 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-camp".freeze
-  s.version = "0.1.4".freeze
+  s.version = "0.1.6".freeze
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Miguel Vazquez".freeze]
-  s.date = "2025-03-25"
+  s.date = "2025-04-08"
   s.description = "Functionalities to deploy and use scouts in remote servers like AWS".freeze
   s.email = "mikisvaz@gmail.com".freeze
   s.executables = ["scout-camp".freeze]
@@ -38,13 +38,17 @@ Gem::Specification.new do |s|
     "lib/scout/terraform_dsl/util.rb",
     "scout-camp.gemspec",
     "scout_commands/offsite",
-    "scout_commands/terraform/add",
+    "scout_commands/terraform/add/lambda",
+    "scout_commands/terraform/add/relay",
     "scout_commands/terraform/apply",
     "scout_commands/terraform/destroy",
+    "scout_commands/terraform/lambda_task",
     "scout_commands/terraform/list",
     "scout_commands/terraform/plan",
     "scout_commands/terraform/remove",
     "scout_commands/terraform/status",
+    "scout_commands/terraform/task",
+    "share/aws/lambda_function.rb",
     "share/terraform/aws/bucket/main.tf",
     "share/terraform/aws/bucket/output.tf",
     "share/terraform/aws/bucket/variables.tf",
@@ -64,6 +68,8 @@ Gem::Specification.new do |s|
     "share/terraform/aws/role/main.tf",
     "share/terraform/aws/role/output.tf",
     "share/terraform/aws/role/variables.tf",
+    "share/terraform/ssh/cmd/main.tf",
+    "share/terraform/ssh/cmd/variables.tf",
     "test/scout/aws/test_s3.rb",
     "test/scout/offsite/test_ssh.rb",
     "test/scout/offsite/test_step.rb",

data/scout_commands/terraform/{add → add/lambda}

@@ -14,6 +14,7 @@ $ #{$0} [<options>] <name>
 -w--workflows* Workflows to provision
 -w--dependencies* Workflows to provision
 -b--bucket* Bucket to connect
+--pkg* Pkgdir, scout or rbbt (defaults to scout)
 EOF
 if options[:help]
   if defined? scout_usage
@@ -31,37 +32,20 @@ deployments = Scout.var.deployments.glob_all("*").collect(&:basename)
 raise "Deployment clash" if deployments.include? name
 dir =  Scout.var.deployments[name]
 
-IndiferentHash.add_defaults options, workflows: '', dependencies: 'scout-gear,scout-camp', bucket: 'scout.var'
+IndiferentHash.add_defaults options, workflows: '', dependencies: 'scout-gear,scout-camp', bucket: 'scout.var', pkg: 'scout'
 
 workflows = options[:workflows].split(",")
 dependencies = options[:dependencies].split(",")
+dependencies.push 'scout-camp' unless dependencies.include?('scout-camp')
+pkg = options[:pkg]
 
-code =<<-'EOF'
-def lambda_handler(event:, context:)
-  require 'scout'
-  Path.path_maps[:bucket] = "s3://#{ENV["AWS_BUCKET"]}/{TOPLEVEL}/{SUBPATH}"
-  Path.path_maps[:default] = :bucket
-
-  require 'scout/workflow'
-  require 'scout/aws/s3'
-
-  workflow, task_name, jobname, inputs = IndiferentHash.process_options event, 
-  :workflow, :task_name, :jobname, :inputs
-
-  wf = Workflow.require_workflow workflow
-
-  job = wf.job(task_name, jobname, inputs)
-  job.run
-end
-EOF
-
-def lambda_package(dependencies, workflows, code)
+def lambda_package(dependencies, workflows, function_file, pkg)
   TmpFile.with_path do |dir|
-    dir["lambda_function.rb"].write code
+    Open.cp function_file, dir["lambda_function.rb"]
     dir["Gemfile"].write "source 'https://rubygems.org'\n" + dependencies.collect{|d| "gem '#{d}'"}*"\n"
     workflows.each do |workflow|
-      Open.mkdir dir[".scout"].workflows
-      Open.cp Scout.workflows[workflow], dir[".scout"].workflows[workflow]
+      Open.mkdir dir[".#{pkg}"].workflows
+      Open.ln_s Scout.workflows[workflow], dir[".#{pkg}"].workflows[workflow]
     end
     Misc.in_dir dir do
       `bundle config set path 'vendor/bundle'`
@@ -69,19 +53,18 @@ def lambda_package(dependencies, workflows, code)
       `rm -Rf vendor/bundle/ruby/*/cache`
       `rm -Rf vendor/bundle/ruby/*/gems/RubyInline*`
       TmpFile.with_file nil, extension: :zip do |zip|
-        `zip -r "#{zip}" vendor/bundle lambda_function.rb .scout`
+        `zip -r "#{zip}" vendor/bundle lambda_function.rb .#{pkg}`
         yield zip
       end
     end
   end
 end
 
-require 'aws-sdk'
-lambda_package dependencies, workflows, code do |file|
-  terraform = TerraformDSL.new
+lambda_package dependencies, workflows, Scout.share.aws.lambda_function.set_extension('rb'), options[:pkg] do |file|
   filename = 'lambda_package.zip'
   Open.cp file, dir[filename]
 
+  terraform = TerraformDSL.new
   role = terraform.add :aws, :role, role_name: "lambda_execution_role", principal: {Service: ["lambda.amazonaws.com"]}
 
   terraform.add :aws, :policy_attachment, name: "exec_policy", policy_name: "lamda_basic_execution", 

data/scout_commands/terraform/add/relay

@@ -0,0 +1,40 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Add a new deployment
+
+$ #{$0} [<options>] <name>
+
+-h--help Print this help
+-s--server* Server name
+-u--user* User in server
+-m--model* Model name
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+name = ARGV.shift
+raise MissingParameterException, :name if name.nil?
+
+server, user, model = IndiferentHash.process_options options, :server, :user, :model
+
+deployments = Scout.var.deployments.glob_all("*").collect(&:basename)
+raise "Deployment clash" if deployments.include? name
+dir =  Scout.var.deployments[name]
+
+terraform = TerraformDSL.new
+
+ollama = terraform.add :ssh, :cmd, service_id: "#{name}_ollama", name: "ollama", host: server, user: user, command: 'module load ollama; ollama serve'
+terraform.add :ssh, :cmd, service_id: "#{name}_ollama", name: "process", host: server, user: user, command: %Q(bash -l -c "scout-ai llm process -ck 'backend ollama ask,model #{model} ask'"), depends_on: [ollama] 
+terraform.config dir

data/scout_commands/terraform/lambda_task

@@ -0,0 +1,124 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Call a task on AWS
+
+$ #{$0} [<options>] <workflow> <task> [<other|->]*
+
+-h--help Print this help
+--prefix* Prefix, defaults to Scout
+--clean Clean job
+--recursive_clean Clean job recursively
+EOF
+workflow, task_name = ARGV
+
+if workflow.nil? && options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+raise ParamterException, "No workflow specified" if workflow.nil?
+
+prefix, clean, recursive_clean = IndiferentHash.process_options options, :prefix, :clean, :recursive_clean, 
+  prefix: "Scout"
+
+require 'aws-sdk-lambda'
+
+payload = {}
+payload["workflow"] = workflow
+payload["task_name"] = task_name
+
+if clean
+  payload["clean"] = true
+elsif recursive_clean
+  payload["clean"] = 'recursive'
+end
+
+lambda_handler = "#{prefix}Job"
+
+def aws_lambda(name, payload)
+  client = Aws::Lambda::Client.new
+  resp = client.invoke({
+    function_name: name, 
+    payload: payload.to_json, 
+  })
+
+  JSON.load(resp.payload)
+end
+
+def SOPT_str(task_info)
+  sopt_options = []
+  task_info[:inputs].each do |name|
+    options = task_info[:input_options][name]
+    type = task_info[:input_types][name]
+    shortcut = (options && options[:shortcut]) || name.to_s.slice(0,1)
+    boolean = type.to_sym == :boolean
+
+    sopt_options << "-#{shortcut}--#{name}#{boolean ? "" : "*"}"
+  end
+
+  sopt_options * ":"
+end
+
+def get_SOPT(task_info)
+  IndiferentHash.setup task_info
+  sopt_option_string = self.SOPT_str(task_info)
+  job_options = SOPT.get sopt_option_string
+
+  task_info[:inputs].each do |name|
+    options = task_info[:input_options][name]
+    type = task_info[:input_types][name]
+    next unless type.to_s.include?('array')
+    if job_options.include?(name) && (! Open.exist?(job_options[name]) || type.to_s.include?('file') || type.to_s.include?('path'))
+      job_options[name] = job_options[name].split(",")
+    end
+  end
+  job_options
+end
+
+res = case task_name
+      when nil
+
+      if options[:help]
+          if defined? scout_usage
+            scout_usage 
+          else
+            puts SOPT.doc
+          end
+          exit 0
+        end
+
+        aws_lambda(lambda_handler, payload)
+      else
+        task_info = aws_lambda(lambda_handler, payload.merge(task_name: "info", inputs: {task_name: task_name}))
+        
+        payload["inputs"] = get_SOPT(task_info)
+
+        if options[:help]
+          if defined? scout_usage
+            scout_usage 
+          else
+            puts SOPT.doc
+          end
+          exit 0
+        end
+
+        aws_lambda(lambda_handler, payload)
+      end
+
+if Hash === res && res["errorMessage"]
+  Log.error res["errorMessage"]
+  Log.stack res["stackTrace"] if res["stackTrace"]
+  exit -1
+else
+  iii res
+end

data/scout_commands/terraform/list

@@ -11,6 +11,7 @@ List all deployments
 $ #{$0} [<options>] <filename>
 
 -h--help Print this help
+-a--active Show if they are active
 EOF
 if options[:help]
   if defined? scout_usage
@@ -26,7 +27,7 @@ deployments = Scout.var.deployments.glob_all("*")
 deployments.each do |dir|
   name = File.basename dir
   deployment = TerraformDSL::Deployment.new dir
-  active = deployment.provisioned_elements.any?
+  active = options[:active] && deployment.provisioned_elements.any?
   
   if active
     puts Log.color(:title, name ) + "\t" + dir + "\t" + "Active" 

data/scout_commands/terraform/task

@@ -0,0 +1,92 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Call a task on AWS
+
+$ #{$0} [<options>] <workflow> <task> [<other|->]*
+
+-h--help Print this help
+--prefix* Prefix, defaults to Scout
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+workflow, task_name = ARGV
+
+raise ParamterException, "No workflow specified" if workflow.nil?
+
+prefix = IndiferentHash.process_options options, :prefix, prefix: "Scout"
+require 'aws-sdk-lambda'
+payload = {}
+payload["workflow"] = workflow
+payload["task_name"] = task_name
+lambda_handler = "#{prefix}Job"
+
+def aws_lambda(name, payload)
+  client = Aws::Lambda::Client.new
+  resp = client.invoke({
+    function_name: name, 
+    payload: payload.to_json, 
+  })
+
+  JSON.load(resp.payload)
+end
+
+def SOPT_str(task_info)
+  sopt_options = []
+  task_info[:inputs].each do |name|
+    options = task_info[:input_options][name]
+    type = task_info[:input_types][name]
+    shortcut = (options && options[:shortcut]) || name.to_s.slice(0,1)
+    boolean = type.to_sym == :boolean
+
+    sopt_options << "-#{shortcut}--#{name}#{boolean ? "" : "*"}"
+  end
+
+  sopt_options * ":"
+end
+
+def get_SOPT(task_info)
+  IndiferentHash.setup task_info
+  sopt_option_string = self.SOPT_str(task_info)
+  job_options = SOPT.get sopt_option_string
+
+  task_info[:inputs].each do |name|
+    options = task_info[:input_options][name]
+    type = task_info[:input_types][name]
+    next unless type.to_s.include?('array')
+    if job_options.include?(name) && (! Open.exist?(job_options[name]) || type.to_s.include?('file') || type.to_s.include?('path'))
+      job_options[name] = job_options[name].split(",")
+    end
+  end
+  job_options
+end
+
+res = case task_name
+      when nil
+        aws_lambda(lambda_handler, payload)
+      else
+        task_info = aws_lambda(lambda_handler, payload.merge(task_name: "info", inputs: {task_name: task_name}))
+        
+        payload["inputs"] = get_SOPT(task_info)
+
+        aws_lambda(lambda_handler, payload)
+      end
+
+if res["errorMessage"]
+  Log.error res["errorMessage"]
+  Log.stack res["stackTrace"]
+else
+  iii res
+end

data/share/aws/lambda_function.rb

@@ -0,0 +1,27 @@
+def lambda_handler(event:, context:)
+  require 'scout'
+
+  Path.path_maps[:bucket] = "s3://#{ENV["AWS_BUCKET"]}/{TOPLEVEL}/{SUBPATH}"
+  Path.path_maps[:default] = :bucket
+
+  require 'scout/workflow'
+  require 'scout/aws/s3'
+
+  workflow, task_name, jobname, inputs = IndiferentHash.process_options event, 
+  :workflow, :task_name, :jobname, :inputs
+  
+  raise ParamterException, "No workflow specified" if workflow.nil?
+  
+  workflow = Workflow.require_workflow workflow
+
+  case task_name
+  when nil
+    return {tasks: workflow.tasks.keys, documentation: workflow.documentation}
+  when "info"
+    raise ParamterException, "No task_name specified" if task_name.nil?
+    return workflow.task_info(inputs["task_name"])
+  else
+    job = workflow.job(task_name, jobname, inputs)
+    job.run
+  end
+end

data/share/terraform/aws/host/variables.tf

@@ -1,4 +1,3 @@
-
 variable "ami" {
   description = "AMI id for host"
   type = string

data/share/terraform/ssh/cmd/main.tf

@@ -0,0 +1,34 @@
+terraform {
+  required_providers {
+    ssh = {
+      source  = "loafoe/ssh"
+      version = ">= 1.0.0"
+    }
+  }
+}
+
+resource "ssh_resource" "up" {
+  host = var.host
+  user = var.user
+  agent = true
+  when = "create"
+
+  commands = [
+    "${var.command} > ${var.service_id}.log 2>&1 & echo $! > /tmp/${var.service_id}.pid"
+  ]
+
+}
+
+resource "ssh_resource" "down" {
+  host = var.host
+  user = var.user
+  agent = true
+  when = "destroy"
+
+  commands = [
+    "kill $(cat /tmp/${var.service_id}.pid) || true",
+    "rm -f /tmp/${var.service_id}.pid"
+  ]
+}
+
+

data/share/terraform/ssh/cmd/variables.tf

@@ -0,0 +1,19 @@
+variable "host" {
+  description = "Target host for SSH"
+  type        = string
+}
+
+variable "user" {
+  description = "Username for SSH"
+  type        = string
+}
+
+variable "service_id" {
+  description = "Identifier of the service, should be unique to avoid collisions"
+  type        = string
+}
+
+variable "command" {
+  description = "Command to execute"
+  type        = string
+}

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: scout-camp
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.6
 platform: ruby
 authors:
 - Miguel Vazquez
 bindir: bin
 cert_chain: []
-date: 2025-03-25 00:00:00.000000000 Z
+date: 2025-04-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: scout-essentials
@@ -50,13 +50,17 @@ files:
 - lib/scout/terraform_dsl/util.rb
 - scout-camp.gemspec
 - scout_commands/offsite
-- scout_commands/terraform/add
+- scout_commands/terraform/add/lambda
+- scout_commands/terraform/add/relay
 - scout_commands/terraform/apply
 - scout_commands/terraform/destroy
+- scout_commands/terraform/lambda_task
 - scout_commands/terraform/list
 - scout_commands/terraform/plan
 - scout_commands/terraform/remove
 - scout_commands/terraform/status
+- scout_commands/terraform/task
+- share/aws/lambda_function.rb
 - share/terraform/aws/bucket/main.tf
 - share/terraform/aws/bucket/output.tf
 - share/terraform/aws/bucket/variables.tf
@@ -76,6 +80,8 @@ files:
 - share/terraform/aws/role/main.tf
 - share/terraform/aws/role/output.tf
 - share/terraform/aws/role/variables.tf
+- share/terraform/ssh/cmd/main.tf
+- share/terraform/ssh/cmd/variables.tf
 - test/scout/aws/test_s3.rb
 - test/scout/offsite/test_ssh.rb
 - test/scout/offsite/test_step.rb