scout-camp 0.1.10 → 0.1.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: de2b027b60ec3f0c33cb643258e365f129b217368040cc49d6df0b7232d643f9
-  data.tar.gz: a89e3ed2424d5eb99f107314bab8f6406297b1e2961d6b7fb084f652abcb4705
+  metadata.gz: aca4938597e0fc0bed97dbb0aa2b88821a71c27dfd3b4d96ec9876acaee294be
+  data.tar.gz: c6d270e7f0b8fb479f6032cb39643b54c4e7182444746b52eb78e0875b7355f0
 SHA512:
-  metadata.gz: 30a2f0f6b5f1c28dfb6b2c364c78e8efa10c1dca04ad5a53ca651c62607c41ec74981a4729e885b778954e67dd5be4fcd6c27b12ea976bfc71854c55f97a15f7
-  data.tar.gz: 5c598b2cc4295153b2c2ad8cd7105c7d9b4dfb37b6a176065520523159e8de67167b5a211d06bcdef8322bd3f59b3414734ef1ac380fd80996ebf1fe07a49600
+  metadata.gz: b178c0cf9038d614f0a0c4f19a8d03ea3b5f18194c803d13eb98ee3e06fa818568e0796d094c37402ae8396ddfb861d15fb9a91aef184cd1ce6180daa95618c8
+  data.tar.gz: db7cb1136f29a209b8e64ef2953132bc0de7f87a5484efe2d9f3f59eb263020c8e7ceff53f51f919568981cf9ffc5691fe4e8880dc59b55ecc6fdfbcb4925bdb

data/.vimproject

@@ -47,10 +47,22 @@ scout-camp=/$PWD filter="*" {
  share=share {
   terraform=terraform {
    aws=aws {
+    event_bridge=event_bridge{
+     main.tf
+     data.tf
+     variables.tf
+    }
+
+    lambda=lambda {
+     main.tf
+     variables.tf
+    }
+
     fargate=fargate{
      main.tf
      variables.tf
      locals.tf
+     output.tf
     }
 
     efs_host=efs_host{
@@ -82,13 +94,9 @@ scout-camp=/$PWD filter="*" {
      variables.tf
     }
 
-    lambda=lambda {
-     main.tf
-     variables.tf
-    }
-
     container_lambda=container_lambda{
      main.tf
+     data.tf
      locals.tf
      variables.tf
     }

data/VERSION

@@ -1 +1 @@
-0.1.10
+0.1.12

data/lib/scout/aws/s3.rb

@@ -21,14 +21,18 @@ module Open
       uri.start_with? 's3://'
     end
 
-    def self.claim(uri, ...)
+    def self.claim_uri(uri)
       if Path === uri and not uri.located?
-        is_s3? uri.find
+        is_s3?(uri.find)
       else
         is_s3? uri
       end
     end
 
+    def self.claim(uri, uri2=nil, ...)
+      claim_uri(uri) || (String === uri2 && claim_uri(uri2))
+    end
+
     def self.parse_s3_uri(uri)
       uri = uri.find if Path === uri and not uri.located?
       uri = uri.sub(%r{^s3://}, '')
@@ -66,13 +70,12 @@ module Open
       end
     end
 
-    def self.glob(uri, pattern="**/*")
+    def self.glob(uri, pattern="*")
       bucket, prefix = parse_s3_uri(uri)
       s3 = Aws::S3::Client.new
       matches = []
       continuation_token = nil
 
-      Log.debug "Glob: #{uri} #{pattern}"
       loop do
         resp = s3.list_objects_v2(
           bucket: bucket,
@@ -156,20 +159,29 @@ module Open
 
     def self.cp(source, target)
       if is_s3?(target)
-        source_bucket, source_key = parse_s3_uri(source)
-        target_bucket, target_key = parse_s3_uri(target)
-
-        s3 = Aws::S3::Client.new
-        s3.copy_object({
-          copy_source: "#{source_bucket}/#{source_key}",
-          bucket: target_bucket,
-          key: target_key
-        })
+        if is_s3?(source)
+          source_bucket, source_key = parse_s3_uri(source)
+          target_bucket, target_key = parse_s3_uri(target)
+
+          s3 = Aws::S3::Client.new
+          s3.copy_object({
+            copy_source: "#{source_bucket}/#{source_key}",
+            bucket: target_bucket,
+            key: target_key
+          })
+        else
+          self.write(target, Open.get_stream(source))
+        end
       else
         Open.sensible_write(target, get_stream(source))
       end
     end
 
+    def self.mv(source, target)
+      self.cp(source, target)
+      Open.rm_rf source
+    end
+
     def self.file_exists?(uri)
       bucket, key = parse_s3_uri(uri)
       return false if key.empty? # Can't check existence of bucket this way
@@ -185,6 +197,7 @@ module Open
       bucket, key = parse_s3_uri(uri)
       return false if key.empty? # Can't check existence of bucket this way
 
+      key += '/' unless key.end_with?('/')
       s3 = Aws::S3::Client.new
       response = s3.list_objects_v2({
         bucket: bucket,
@@ -224,6 +237,68 @@ module Open
     def self.ln_s(source, target, options = {})
       cp(source, target)
     end
+
+    def self.sync(source, target, options = {})
+      excludes, files, hard_link, test, print, delete, other = IndiferentHash.process_options options,
+        :excludes, :files, :hard_link, :test, :print, :delete, :other
+
+      excludes ||= %w(.save .crap .source tmp filecache open-remote)
+      excludes = excludes.split(/,\s*/) if excludes.is_a?(String) and not excludes.include?("--exclude")
+
+      if File.directory?(source) || source.end_with?("/")
+        source += "/" unless source.end_with? '/'
+        target += "/" unless target.end_with? '/'
+      end
+
+      if source == target
+        Log.warn "Asking to sync with itself"
+        return
+      end
+
+      Log.low "Migrating #{source} #{files.length} files to #{target} - #{Misc.fingerprint(files)}}" if files
+
+      sync_args = %w()
+      sync_args << excludes.collect{|s| "--exclude '#{s}'" } if excludes and excludes.any?
+      sync_args << "-nv" if test
+
+      if files
+        tmp_files = TmpFile.tmp_file 's3_sync_files-'
+        Open.write(tmp_files, files * "\n")
+        sync_args << "--files-from='#{tmp_files}'"
+      end
+
+      if Open.directory?(source)
+        cmd = "aws s3 sync #{sync_args * " "} #{source} #{target}"
+      else
+        cmd = "aws s3 cp #{source} #{target}"
+      end
+      case other
+      when String
+        cmd << " " << other
+      when Array
+        cmd << " " << other * " "
+      end
+      cmd << " && rm -Rf #{source}" if delete && ! files
+
+      if print
+        cmd
+      else
+        CMD.cmd_log(cmd, :log => Log::HIGH)
+
+        if delete && files
+          remove_files = files.collect{|f| File.join(source, f) }
+          dirs = remove_files.select{|f| File.directory? f }
+          remove_files.each do |file|
+            next if dirs.include? file
+            Open.rm file
+          end
+
+          dirs.each do |dir|
+            FileUtils.rmdir dir if Dir.glob(dir).empty?
+          end
+        end
+      end
+    end
   end
 end
 

data/lib/scout/offsite/resource.rb

@@ -11,7 +11,7 @@ module Resource
       resource = path.pkgdir if resource.nil? and path.is_a?(Path) and path.pkgdir.is_a?(Resource)
       resource = Resource.default_resource if resource.nil?
 
-      if File.exist?(path)
+      if Path.located?(path)
         real_paths = [path]
       else
         path = Path.setup(path, pkgdir: resource) unless path.is_a?(Path)

data/scout-camp.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-camp 0.1.10 ruby lib
+# stub: scout-camp 0.1.12 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-camp".freeze
-  s.version = "0.1.10".freeze
+  s.version = "0.1.12".freeze
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Miguel Vazquez".freeze]
-  s.date = "2025-04-16"
+  s.date = "1980-01-02"
   s.description = "Functionalities to deploy and use scouts in remote servers like AWS".freeze
   s.email = "mikisvaz@gmail.com".freeze
   s.executables = ["scout-camp".freeze]
@@ -72,8 +72,12 @@ Gem::Specification.new do |s|
     "share/terraform/aws/efs_host/main.tf",
     "share/terraform/aws/efs_host/output.tf",
     "share/terraform/aws/efs_host/variables.tf",
+    "share/terraform/aws/event_bridge/data.tf",
+    "share/terraform/aws/event_bridge/main.tf",
+    "share/terraform/aws/event_bridge/variables.tf",
     "share/terraform/aws/fargate/locals.tf",
     "share/terraform/aws/fargate/main.tf",
+    "share/terraform/aws/fargate/output.tf",
     "share/terraform/aws/fargate/variables.tf",
     "share/terraform/aws/host/locals.tf",
     "share/terraform/aws/host/main.tf",
@@ -112,7 +116,7 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://github.com/mikisvaz/scout-camp".freeze
   s.licenses = ["MIT".freeze]
-  s.rubygems_version = "3.6.6".freeze
+  s.rubygems_version = "3.6.7".freeze
   s.summary = "Deploy you scouts".freeze
 
   s.specification_version = 4

data/scout_commands/sync

@@ -2,6 +2,7 @@
 
 require 'scout'
 require 'scout/offsite/resource'
+require 'scout/aws/s3'
 
 $0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
 

data/scout_commands/terraform/lambda_task

@@ -14,6 +14,7 @@ $ #{$0} [<options>] <workflow> <task> [<other|->]*
 --prefix* Prefix, defaults to Scout
 --queue Queue job
 --clean Clean job
+--info Return job info
 --recursive_clean Clean job recursively
 EOF
 workflow, task_name = ARGV
@@ -27,9 +28,9 @@ if workflow.nil? && options[:help]
   exit 0
 end
 
-raise ParamterException, "No workflow specified" if workflow.nil?
+raise ParameterException, "No workflow specified" if workflow.nil?
 
-prefix, clean, recursive_clean, queue = IndiferentHash.process_options options, :prefix, :clean, :recursive_clean, :queue,
+prefix, clean, recursive_clean, queue, info = IndiferentHash.process_options options, :prefix, :clean, :recursive_clean, :queue, :info,
   prefix: "Scout"
 
 require 'aws-sdk-lambda'
@@ -116,6 +117,8 @@ res = case task_name
           exit 0
         end
 
+        payload["info"] = true if info
+
         aws_lambda(lambda_handler, payload)
       end
 

data/share/aws/lambda_function.rb

@@ -7,8 +7,13 @@ def lambda_handler(event:, context:)
   require 'scout/workflow'
   require 'scout/aws/s3'
 
-  workflow, task_name, jobname, inputs, clean, queue = IndiferentHash.process_options event,
-    :workflow, :task_name, :jobname, :inputs, :clean, :queue
+  TmpFile.tmpdir = Path.setup('/tmp')
+  Open.sensible_write_dir = Path.setup('/tmp/sensible_write')
+
+  Log.info "Payload: #{Log.fingerprint(event)}"
+
+  workflow, task_name, jobname, inputs, clean, queue, info = IndiferentHash.process_options event,
+    :workflow, :task_name, :jobname, :inputs, :clean, :queue, :info
 
   raise ParameterException, "No workflow specified" if workflow.nil?
 
@@ -21,6 +26,8 @@ def lambda_handler(event:, context:)
     raise ParameterException, "No task_name specified" if task_name.nil?
     return workflow.task_info(inputs["task_name"])
   else
+    Workflow.job_cache.clear
+
     job = workflow.job(task_name, jobname, inputs)
 
     case clean
@@ -30,7 +37,11 @@ def lambda_handler(event:, context:)
       job.recursive_clean
     end
 
-    if job.done?
+    if info
+      info = job.info.dup
+      info["path"] = job.path
+      info
+    elsif job.done?
       job.load
     elsif job.error?
       raise job.exception
@@ -40,8 +51,8 @@ def lambda_handler(event:, context:)
         body: job.path
       }
     elsif queue
-      save_inputs = Scout.var.queue[workflow.to_s][task_name][job.name].find
-      job.save_inputs(save_inputs)
+      save_inputs = Scout.var.queue[workflow.to_s][task_name][job.name].find :bucket
+      job.save_input_bundle(save_inputs) unless save_inputs.exists?
       {
         statusCode: 202,
         body: job.path

data/share/terraform/aws/event_bridge/data.tf

@@ -0,0 +1,10 @@
+data "aws_vpc" "default" {
+  default = true
+}
+
+data "aws_subnets" "default" {
+  filter {
+    name   = "vpc-id"
+    values = [data.aws_vpc.default.id]
+  }
+}

data/share/terraform/aws/event_bridge/main.tf

@@ -0,0 +1,85 @@
+# IAM Role for EventBridge to trigger ECS task
+resource "aws_iam_role" "eventbridge_invoke_ecs" {
+  name = "eventbridge_invoke_ecs"
+
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17",
+    Statement = [
+      {
+        Effect = "Allow",
+        Principal = {
+          Service = "events.amazonaws.com"
+        },
+        Action = "sts:AssumeRole"
+      }
+    ]
+  })
+}
+
+# Permissions for EventBridge to run the task
+resource "aws_iam_role_policy" "ecs_task_invoke_policy" {
+  role = aws_iam_role.eventbridge_invoke_ecs.name
+
+  policy = jsonencode({
+    Version = "2012-10-17",
+    Statement = [
+      {
+        Effect = "Allow",
+        Action = [
+          "ecs:RunTask"
+        ],
+        Resource = "*"
+      },
+      {
+        Effect = "Allow",
+        Action = [
+          "iam:PassRole"
+        ],
+        Resource = "*"
+      }
+    ]
+  })
+}
+
+# EventBridge rule triggered on S3 put event
+resource "aws_cloudwatch_event_rule" "s3_put_event" {
+  name        = "s3-put-to-uploads"
+  description = "Triggers Fargate task on S3 file upload"
+  event_pattern = jsonencode({
+    source = ["aws.s3"],
+    "detail-type" = ["Object Created"],
+    detail = {
+      bucket = {
+        name = [var.bucket]
+      },
+      object = {
+        key = [{
+          prefix = var.directory
+        }]
+      }
+    }
+  })
+}
+
+# Target: ECS Fargate task
+resource "aws_cloudwatch_event_target" "run_fargate" {
+  rule      = aws_cloudwatch_event_rule.s3_put_event.name
+  arn       = var.cluster_arn # Replace with your ECS Cluster ARN
+  role_arn  = aws_iam_role.eventbridge_invoke_ecs.arn
+
+  ecs_target {
+    task_definition_arn = var.task_arn
+    launch_type         = "FARGATE"
+    network_configuration {
+      assign_public_ip = true
+      subnets          = data.aws_subnets.default.ids
+    }
+  }
+}
+
+# Allow S3 to send events to EventBridge
+#resource "aws_s3_bucket_notification" "s3_event" {
+#  bucket = var.bucket_name
+#
+#  eventbridge = true
+#}

data/share/terraform/aws/event_bridge/variables.tf

@@ -0,0 +1,19 @@
+# Replace with your actual bucket
+variable "bucket" {
+  type = string
+}
+
+variable "directory" {
+  type = string
+}
+
+variable "task_arn" {
+  type = string
+}
+
+variable "cluster_arn" {
+  type = string
+}
+
+
+

data/share/terraform/aws/fargate/output.tf

@@ -0,0 +1,10 @@
+output "task_arn" {
+  description = "Task arn"
+  value = aws_ecs_task_definition.this.arn
+}
+
+output "cluster_arn" {
+  description = "Task arn"
+  value = aws_ecs_cluster.this.arn
+}
+

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: scout-camp
 version: !ruby/object:Gem::Version
-  version: 0.1.10
+  version: 0.1.12
 platform: ruby
 authors:
 - Miguel Vazquez
 bindir: bin
 cert_chain: []
-date: 2025-04-16 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: scout-essentials
@@ -84,8 +84,12 @@ files:
 - share/terraform/aws/efs_host/main.tf
 - share/terraform/aws/efs_host/output.tf
 - share/terraform/aws/efs_host/variables.tf
+- share/terraform/aws/event_bridge/data.tf
+- share/terraform/aws/event_bridge/main.tf
+- share/terraform/aws/event_bridge/variables.tf
 - share/terraform/aws/fargate/locals.tf
 - share/terraform/aws/fargate/main.tf
+- share/terraform/aws/fargate/output.tf
 - share/terraform/aws/fargate/variables.tf
 - share/terraform/aws/host/locals.tf
 - share/terraform/aws/host/main.tf
@@ -139,7 +143,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.6
+rubygems_version: 3.6.7
 specification_version: 4
 summary: Deploy you scouts
 test_files: []