scout-camp 0.1.6 → 0.1.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dd5f35a6409cd8556f8fb6cb8641c28c36fb3b0dcc9110eeef8c8d9a5ea42594
-  data.tar.gz: 7bb5e7f5b07ac7d9ca601909bc3c5632d6f9d1630020e308d901d8b3a771d0f5
+  metadata.gz: de2b027b60ec3f0c33cb643258e365f129b217368040cc49d6df0b7232d643f9
+  data.tar.gz: a89e3ed2424d5eb99f107314bab8f6406297b1e2961d6b7fb084f652abcb4705
 SHA512:
-  metadata.gz: e460301e94edf73163f57309498db7d04b38b202c0437938dc0bf10cb0a31e1683f8d908309b0876fe106e781f2b69e24563656562eb57e0a759116bfb38f904
-  data.tar.gz: 6a78670a91b259399cad19a95411c214f3382173c37bb2611f4b13deb8a32a1626d0f2dfb793e8462253e6322156b104816a0bc5f3f37ce9d0cf3017bf735adf
+  metadata.gz: 30a2f0f6b5f1c28dfb6b2c364c78e8efa10c1dca04ad5a53ca651c62607c41ec74981a4729e885b778954e67dd5be4fcd6c27b12ea976bfc71854c55f97a15f7
+  data.tar.gz: 5c598b2cc4295153b2c2ad8cd7105c7d9b4dfb37b6a176065520523159e8de67167b5a211d06bcdef8322bd3f59b3414734ef1ac380fd80996ebf1fe07a49600

data/.vimproject

@@ -19,19 +19,24 @@ scout-camp=/$PWD filter="*" {
     ssh.rb
     step.rb
     sync.rb
+    resource.rb
    }
   }
  }
  scout_commands=scout_commands {
   offsite
+  sync
+
   terraform=terraform{
    list
    add=add{
-    lambda
     relay
+    lambda
+    host
+    fargate
    }
-   add
    status
+   outputs
    apply
    plan
    destroy
@@ -42,38 +47,89 @@ scout-camp=/$PWD filter="*" {
  share=share {
   terraform=terraform {
    aws=aws {
+    fargate=fargate{
+     main.tf
+     variables.tf
+     locals.tf
+    }
+
+    efs_host=efs_host{
+     main.tf
+     variables.tf
+     locals.tf
+     data.tf
+     output.tf
+    }
+
+    efs=efs{
+     main.tf
+     variables.tf
+     data.tf
+     locals.tf
+     output.tf
+    }
+
+    network=network{
+     data.tf
+     main.tf
+     variables.tf
+     output.tf
+    }
+
     bucket=bucket {
      main.tf
      output.tf
      variables.tf
     }
+
     lambda=lambda {
      main.tf
      variables.tf
     }
+
+    container_lambda=container_lambda{
+     main.tf
+     locals.tf
+     variables.tf
+    }
+
     role=role{
      main.tf
      variables.tf
      output.tf
     }
+    
+    policy=policy{
+     main.tf
+     variables.tf
+     output.tf
+    }
+
+    role_policy=role_policy{
+     main.tf
+     variables.tf
+    }
+
     policy_attachment=policy_attachment {
      main.tf
      variables.tf
     }
-    cluster=cluster {
+
+    iam_instance_profile=iam_instance_profile{
      main.tf
-     output.tf
      variables.tf
+     output.tf
     }
-    host=host {
-     locals.tf
+
+    cluster=cluster {
      main.tf
      output.tf
      variables.tf
     }
-    fargate=fargate{
+    host=host {
      locals.tf
      main.tf
+     output.tf
      variables.tf
     }
     provider=provider {

data/VERSION

@@ -1 +1 @@
-0.1.6
+0.1.10

data/lib/scout/aws/s3.rb

@@ -59,12 +59,20 @@ module Open
       s3.put_object(bucket: bucket, key: key, body: content)
     end
 
+    def self.touch(uri)
+      if self.exists?(uri)
+      else
+        self.cp(uri, uri)
+      end
+    end
+
     def self.glob(uri, pattern="**/*")
       bucket, prefix = parse_s3_uri(uri)
       s3 = Aws::S3::Client.new
       matches = []
       continuation_token = nil
 
+      Log.debug "Glob: #{uri} #{pattern}"
       loop do
         resp = s3.list_objects_v2(
           bucket: bucket,
@@ -82,8 +90,19 @@ module Open
             remaining = remaining.sub(%r{^/}, '')
           end
 
+          Log.debug "Glob: #{remaining}"
+
           if File.fnmatch?(pattern, remaining, File::FNM_PATHNAME)
             matches << "s3://#{bucket}/#{key}"
+          else
+            dir = File.dirname(remaining)
+            while dir 
+              if File.fnmatch?(pattern, dir, File::FNM_PATHNAME)
+                matches << "s3://#{bucket}/#{File.join(prefix,dir)}"
+              end
+              break if dir == File.dirname(dir)
+              dir = File.dirname(dir)
+            end
           end
         end
 
@@ -136,18 +155,22 @@ module Open
     end
 
     def self.cp(source, target)
-      source_bucket, source_key = parse_s3_uri(source)
-      target_bucket, target_key = parse_s3_uri(target)
+      if is_s3?(target)
+        source_bucket, source_key = parse_s3_uri(source)
+        target_bucket, target_key = parse_s3_uri(target)
 
-      s3 = Aws::S3::Client.new
-      s3.copy_object({
-        copy_source: "#{source_bucket}/#{source_key}",
-        bucket: target_bucket,
-        key: target_key
-      })
+        s3 = Aws::S3::Client.new
+        s3.copy_object({
+          copy_source: "#{source_bucket}/#{source_key}",
+          bucket: target_bucket,
+          key: target_key
+        })
+      else
+        Open.sensible_write(target, get_stream(source))
+      end
     end
 
-    def self.exists?(uri)
+    def self.file_exists?(uri)
       bucket, key = parse_s3_uri(uri)
       return false if key.empty? # Can't check existence of bucket this way
 
@@ -157,6 +180,28 @@ module Open
     rescue Aws::S3::Errors::NotFound, Aws::S3::Errors::NoSuchBucket
       false
     end
+    
+    def self.directory?(uri)
+      bucket, key = parse_s3_uri(uri)
+      return false if key.empty? # Can't check existence of bucket this way
+
+      s3 = Aws::S3::Client.new
+      response = s3.list_objects_v2({
+        bucket: bucket,
+        prefix: key,
+        max_keys: 1
+      })
+
+      !response.contents.empty?
+    end
+
+    def self.exists?(uri)
+      begin
+        file_exists?(uri) || directory?(uri)
+      rescue
+        false
+      end
+    end
 
     self.singleton_class.alias_method :exist?, :exists?
 
@@ -202,6 +247,3 @@ end
 
 Hook.apply(Open::S3, Open)
 Hook.apply(Path::S3, Path)
-
-
-#$ ask -t code --file /home/miki/git/scout-camp/lib/scout/aws/s3.rb extend this file [[...]] to include a function called self.exists? that determines if a uri exists {{{

data/lib/scout/offsite/resource.rb

@@ -0,0 +1,34 @@
+require 'scout/resource'
+require_relative 'sync'
+module Resource
+  def self.sync(path, map = nil, source: nil, target: nil, resource: nil, **kwargs)
+    map = 'user' if map.nil?
+
+    if source
+      paths = [path]
+      real_paths, identified_paths = SSHLine.locate(source, paths)
+    else
+      resource = path.pkgdir if resource.nil? and path.is_a?(Path) and path.pkgdir.is_a?(Resource)
+      resource = Resource.default_resource if resource.nil?
+
+      if File.exist?(path)
+        real_paths = [path]
+      else
+        path = Path.setup(path, pkgdir: resource) unless path.is_a?(Path)
+        real_paths = path.directory? ? path.find_all : path.glob_all
+      end
+      
+      identified_paths = real_paths.collect{|path| resource.identify(path) }
+    end
+
+    if target
+      target_paths = SSHLine.locate(target, identified_paths, map: map)
+    else
+      target_paths = identified_paths.collect{|p| p.find(map) }
+    end
+
+    real_paths.zip(target_paths).each do |source_path,target_path|
+      Open.sync(source_path, target_path, kwargs.merge(source: source, target: target))
+    end
+  end
+end

data/lib/scout/offsite/ssh.rb

@@ -1,4 +1,5 @@
 require 'net/ssh'
+require 'scout/persist'
 require_relative 'exceptions'
 
 class SSHLine
@@ -90,6 +91,7 @@ class SSHLine
   def scout(script)
     scout_script =<<-EOF
 require 'scout'
+require 'scout/offsite/ssh'
 SSHLine.run_local do
 #{script.strip}
 end

data/lib/scout/offsite/sync.rb

@@ -1,3 +1,4 @@
+require_relative 'ssh'
 class SSHLine
   def self.locate(server, paths, map: :user)
     SSHLine.scout server, <<-EOF
@@ -5,6 +6,7 @@ map = :#{map}
 paths = [#{paths.collect{|p| "'" + p + "'" } * ", " }]
 located = paths.collect{|p| Path.setup(p).find(map) }
 identified = paths.collect{|p| Resource.identify(p) }
+located = located.collect{|path| path << "/" if path.directory? }
 [located, identified]
     EOF
   end

data/lib/scout/terraform_dsl/util.rb

@@ -13,9 +13,9 @@ class TerraformDSL
   # @param prefix [nil,String] Optional prefix to prepend
   def self.log(msg, prefix = nil)
     if prefix
-      STDOUT.puts("[#{prefix}] " + msg)
+      Log.info("[#{prefix}] " + msg)
     else
-      STDOUT.puts(msg)
+      Log.info(msg)
     end
   end
 

data/lib/scout/terraform_dsl.rb

@@ -80,7 +80,17 @@ class TerraformDSL
     def to_json(*_args)
       ['module', @name].join('.')
     end
+  end
+
+  module DirectReference
+    def to_json(*_args)
+      self
+    end
 
+    def method_missing(elem)
+      new = self + '.' + elem.to_s
+      new.extend DirectReference
+    end
   end
 
   MODULES_DIR = Scout.share.terraform
@@ -129,6 +139,8 @@ class TerraformDSL
 
       if value.is_a?(String) && (m = value.match(/^module\.(.*)\.(.*)/))
         value = Module::Output.new m[1], m[2]
+      elsif value.is_a?(Symbol)
+        value = variables[value]
       end
 
       acc << "  #{name} = #{value.to_json}"
@@ -180,6 +192,52 @@ class TerraformDSL
     nil
   end
 
+  # Add a backend template file without using modules.
+  #
+  # @param variables [Hash] variables for the backend:
+  #   :bucket, :key & :region
+  def backend(type, variables = {})
+    variables = variables.dup
+
+    text =<<~EOF
+    terraform {
+        backend "#{type}" {
+        #{variable_block(variables)}
+        }
+    }
+    EOF
+
+    element_file = ['backend_config', type.to_s].join('.')
+
+    custom(element_file, text)
+
+    nil
+  end
+
+  # Connect to a remote state through a backend
+  #
+  # @param variables [Hash] variables for the backend:
+  #   :bucket, :key & :region
+  def remote(type, key, variables = {})
+    variables = variables.dup
+    variables[:key] = key unless variables.include?(:key)
+
+    text =<<~EOF
+    data "terraform_remote_state" "#{key}" {
+        backend = "#{type}"
+        config = {
+          #{variable_block(variables)}
+        }
+    }
+    EOF
+
+    element_file = ['remote', type.to_s, key.to_s].join('.')
+
+    custom(element_file, text)
+
+    "data.terraform_remote_state.#{key}".extend DirectReference 
+  end
+
   # Add a provider template file without using modules.
   # Defining providers in modules is problematic when providers
   # are not managed by Hashicorp. Hopefully we can

data/lib/scout-camp.rb

@@ -4,3 +4,4 @@ require 'scout/resource'
 Path.add_path :scout_camp, File.join(Path.caller_lib_dir(__FILE__), "{TOPLEVEL}/{SUBPATH}")
 require 'scout/terraform_dsl'
 require 'scout/offsite'
+require 'scout/aws/s3'

data/scout-camp.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-camp 0.1.6 ruby lib
+# stub: scout-camp 0.1.10 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-camp".freeze
-  s.version = "0.1.6".freeze
+  s.version = "0.1.10".freeze
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Miguel Vazquez".freeze]
-  s.date = "2025-04-08"
+  s.date = "2025-04-16"
   s.description = "Functionalities to deploy and use scouts in remote servers like AWS".freeze
   s.email = "mikisvaz@gmail.com".freeze
   s.executables = ["scout-camp".freeze]
@@ -30,6 +30,7 @@ Gem::Specification.new do |s|
     "lib/scout/aws/s3.rb",
     "lib/scout/offsite.rb",
     "lib/scout/offsite/exceptions.rb",
+    "lib/scout/offsite/resource.rb",
     "lib/scout/offsite/ssh.rb",
     "lib/scout/offsite/step.rb",
     "lib/scout/offsite/sync.rb",
@@ -38,12 +39,14 @@ Gem::Specification.new do |s|
     "lib/scout/terraform_dsl/util.rb",
     "scout-camp.gemspec",
     "scout_commands/offsite",
+    "scout_commands/sync",
     "scout_commands/terraform/add/lambda",
     "scout_commands/terraform/add/relay",
     "scout_commands/terraform/apply",
     "scout_commands/terraform/destroy",
     "scout_commands/terraform/lambda_task",
     "scout_commands/terraform/list",
+    "scout_commands/terraform/outputs",
     "scout_commands/terraform/plan",
     "scout_commands/terraform/remove",
     "scout_commands/terraform/status",
@@ -55,12 +58,39 @@ Gem::Specification.new do |s|
     "share/terraform/aws/cluster/main.tf",
     "share/terraform/aws/cluster/output.tf",
     "share/terraform/aws/cluster/variables.tf",
+    "share/terraform/aws/container_lambda/data.tf",
+    "share/terraform/aws/container_lambda/locals.tf",
+    "share/terraform/aws/container_lambda/main.tf",
+    "share/terraform/aws/container_lambda/variables.tf",
+    "share/terraform/aws/efs/data.tf",
+    "share/terraform/aws/efs/locals.tf",
+    "share/terraform/aws/efs/main.tf",
+    "share/terraform/aws/efs/output.tf",
+    "share/terraform/aws/efs/variables.tf",
+    "share/terraform/aws/efs_host/data.tf",
+    "share/terraform/aws/efs_host/locals.tf",
+    "share/terraform/aws/efs_host/main.tf",
+    "share/terraform/aws/efs_host/output.tf",
+    "share/terraform/aws/efs_host/variables.tf",
+    "share/terraform/aws/fargate/locals.tf",
+    "share/terraform/aws/fargate/main.tf",
+    "share/terraform/aws/fargate/variables.tf",
     "share/terraform/aws/host/locals.tf",
     "share/terraform/aws/host/main.tf",
     "share/terraform/aws/host/output.tf",
     "share/terraform/aws/host/variables.tf",
+    "share/terraform/aws/iam_instance_profile/main.tf",
+    "share/terraform/aws/iam_instance_profile/output.tf",
+    "share/terraform/aws/iam_instance_profile/variables.tf",
     "share/terraform/aws/lambda/main.tf",
     "share/terraform/aws/lambda/variables.tf",
+    "share/terraform/aws/network/data.tf",
+    "share/terraform/aws/network/main.tf",
+    "share/terraform/aws/network/output.tf",
+    "share/terraform/aws/network/variables.tf",
+    "share/terraform/aws/policy/main.tf",
+    "share/terraform/aws/policy/output.tf",
+    "share/terraform/aws/policy/variables.tf",
     "share/terraform/aws/policy_attachment/main.tf",
     "share/terraform/aws/policy_attachment/variables.tf",
     "share/terraform/aws/provider/data.tf",
@@ -68,9 +98,12 @@ Gem::Specification.new do |s|
     "share/terraform/aws/role/main.tf",
     "share/terraform/aws/role/output.tf",
     "share/terraform/aws/role/variables.tf",
+    "share/terraform/aws/role_policy/main.tf",
+    "share/terraform/aws/role_policy/variables.tf",
     "share/terraform/ssh/cmd/main.tf",
     "share/terraform/ssh/cmd/variables.tf",
     "test/scout/aws/test_s3.rb",
+    "test/scout/offsite/test_resource.rb",
     "test/scout/offsite/test_ssh.rb",
     "test/scout/offsite/test_step.rb",
     "test/scout/offsite/test_sync.rb",
@@ -79,7 +112,7 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://github.com/mikisvaz/scout-camp".freeze
   s.licenses = ["MIT".freeze]
-  s.rubygems_version = "3.6.5".freeze
+  s.rubygems_version = "3.6.6".freeze
   s.summary = "Deploy you scouts".freeze
 
   s.specification_version = 4

data/scout_commands/sync

@@ -0,0 +1,33 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+require 'scout/offsite/resource'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Sync to resources between path maps or between hosts
+
+$ #{$0} [<options>] <path> [<path_map>]
+
+Path map defaults to :user
+
+-h--help Print this help
+-s--source* Host of source path
+-t--target* Host of target path
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+path, path_map = ARGV
+
+raise MissingParameterException, :path if path.nil?
+
+Resource.sync(Path.setup(path.dup), path_map, **options)

data/scout_commands/terraform/lambda_task

@@ -12,6 +12,7 @@ $ #{$0} [<options>] <workflow> <task> [<other|->]*
 
 -h--help Print this help
 --prefix* Prefix, defaults to Scout
+--queue Queue job
 --clean Clean job
 --recursive_clean Clean job recursively
 EOF
@@ -28,14 +29,14 @@ end
 
 raise ParamterException, "No workflow specified" if workflow.nil?
 
-prefix, clean, recursive_clean = IndiferentHash.process_options options, :prefix, :clean, :recursive_clean, 
+prefix, clean, recursive_clean, queue = IndiferentHash.process_options options, :prefix, :clean, :recursive_clean, :queue,
   prefix: "Scout"
 
 require 'aws-sdk-lambda'
 
 payload = {}
-payload["workflow"] = workflow
-payload["task_name"] = task_name
+payload[:workflow] = workflow
+payload[:task_name] = task_name
 
 if clean
   payload["clean"] = true
@@ -43,10 +44,13 @@ elsif recursive_clean
   payload["clean"] = 'recursive'
 end
 
+payload["queue"] = true if queue
+
 lambda_handler = "#{prefix}Job"
 
 def aws_lambda(name, payload)
   client = Aws::Lambda::Client.new
+  Log.debug "Sending Lambda #{name} #{Log.fingerprint payload}"
   resp = client.invoke({
     function_name: name, 
     payload: payload.to_json, 
@@ -64,7 +68,7 @@ def SOPT_str(task_info)
     boolean = type.to_sym == :boolean
 
     sopt_options << "-#{shortcut}--#{name}#{boolean ? "" : "*"}"
-  end
+  end if task_info[:inputs]
 
   sopt_options * ":"
 end
@@ -81,7 +85,7 @@ def get_SOPT(task_info)
     if job_options.include?(name) && (! Open.exist?(job_options[name]) || type.to_s.include?('file') || type.to_s.include?('path'))
       job_options[name] = job_options[name].split(",")
     end
-  end
+  end if task_info[:inputs]
   job_options
 end
 

data/scout_commands/terraform/list

@@ -25,12 +25,20 @@ end
 deployments = Scout.var.deployments.glob_all("*")
 
 deployments.each do |dir|
+  next unless File.directory?(dir)
   name = File.basename dir
   deployment = TerraformDSL::Deployment.new dir
-  active = options[:active] && deployment.provisioned_elements.any?
+  #active = options[:active] && deployment.provisioned_elements.any?
   
-  if active
-    puts Log.color(:title, name ) + "\t" + dir + "\t" + "Active" 
+  if options[:active]
+    Log.with_severity 5 do
+      print Log.color(:title, name ) + "\t" + dir + "\t" 
+      if deployment.provisioned_elements.any?
+        puts Log.color :present, "Active" 
+      else
+        puts Log.color :missing, "Inactive" 
+      end
+    end
   else
     puts Log.color(:title, name) + "\t" + dir
   end

data/scout_commands/terraform/outputs

@@ -0,0 +1,33 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Work with deployment
+
+$ #{$0} [<options>] <name>
+
+-h--help Print this help
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+name = ARGV.shift
+raise MissingParameterException, :name if name.nil?
+
+dir = Scout.var.deployments.glob_all(name).first
+
+deployment = TerraformDSL::Deployment.new dir
+deployment.provisioned_elements.each do |element|
+  ppp deployment.element_state(element)
+end
+