scout-camp 0.1.11 → 0.1.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4310a29a500f7fe04fb974b8a5cb3ae5aae506a9a3663663ec6bd86270c17fd2
-  data.tar.gz: a6e9b6a396a76f9ddde67a502658b9cb37cddeb12355c1bbe244c5f1ff7c3f36
+  metadata.gz: b67e89dae365eb1464f34a19d600247a18d9698de5a8337d5cda677b1623396a
+  data.tar.gz: f5f37fce02c5a92ef3d370fd4f1e0367955bcaffc993ba5a963dca94f4ec255c
 SHA512:
-  metadata.gz: 86dbee75e288daf9d1be188a9d13e79b9656314ef3345e24d226fd79fa987c7723c4bd1d2df488a5a2e56e04cdcb97bf69e4a2ef0f2a88bde00713978ff3b4be
-  data.tar.gz: 7d7bd20736cad39b68d9dc0406490a5ea2a5095bec45dfe550fd2e5131ecbbdf4ecfbd21ac523cfcddadd163c4988465ad6358038d38a9c3d5151a48c50cb6cb
+  metadata.gz: 50d5d0ee70db54470257f49ad6b5595beb0b1d5be73df1471088fc336800a075b2d723082834f6dc6252abf1867b95323b84a3e09dd975c6be1ed2a129e2e46a
+  data.tar.gz: 6908c7789074017d48612f538878657dab8165a286ac8468fd5c243549ff1cbe599ef2868e2d5f88ef8df4f859c57aaa95c3fcd9b9603729fd662756c7faefc5

data/.vimproject

@@ -47,10 +47,22 @@ scout-camp=/$PWD filter="*" {
  share=share {
   terraform=terraform {
    aws=aws {
+    event_bridge=event_bridge{
+     main.tf
+     data.tf
+     variables.tf
+    }
+
+    lambda=lambda {
+     main.tf
+     variables.tf
+    }
+
     fargate=fargate{
      main.tf
      variables.tf
      locals.tf
+     output.tf
     }
 
     efs_host=efs_host{
@@ -82,11 +94,6 @@ scout-camp=/$PWD filter="*" {
      variables.tf
     }
 
-    lambda=lambda {
-     main.tf
-     variables.tf
-    }
-
     container_lambda=container_lambda{
      main.tf
      data.tf

data/VERSION

@@ -1 +1 @@
-0.1.11
+0.1.13

data/lib/scout/aws/s3.rb

@@ -65,8 +65,26 @@ module Open
 
     def self.touch(uri)
       if self.exists?(uri)
-      else
         self.cp(uri, uri)
+      else
+        self.write(uri, '')
+      end
+    end
+
+    def self.size(uri)
+      begin
+        bucket, key = parse_s3_uri(uri)
+
+        s3 = Aws::S3::Client.new
+
+        resp = s3.head_object(bucket: bucket, key: key)
+        return resp.content_length
+      rescue Aws::S3::Errors::NotFound
+        puts "Object not found: #{bucket_name}/#{object_key}"
+        return nil
+      rescue => e
+        puts "Error retrieving object size: #{e.message}"
+        return nil
       end
     end
 
@@ -159,20 +177,29 @@ module Open
 
     def self.cp(source, target)
       if is_s3?(target)
-        source_bucket, source_key = parse_s3_uri(source)
-        target_bucket, target_key = parse_s3_uri(target)
-
-        s3 = Aws::S3::Client.new
-        s3.copy_object({
-          copy_source: "#{source_bucket}/#{source_key}",
-          bucket: target_bucket,
-          key: target_key
-        })
+        if is_s3?(source)
+          source_bucket, source_key = parse_s3_uri(source)
+          target_bucket, target_key = parse_s3_uri(target)
+
+          s3 = Aws::S3::Client.new
+          s3.copy_object({
+            copy_source: "#{source_bucket}/#{source_key}",
+            bucket: target_bucket,
+            key: target_key
+          })
+        else
+          self.write(target, Open.get_stream(source))
+        end
       else
         Open.sensible_write(target, get_stream(source))
       end
     end
 
+    def self.mv(source, target)
+      self.cp(source, target)
+      Open.rm_rf source
+    end
+
     def self.file_exists?(uri)
       bucket, key = parse_s3_uri(uri)
       return false if key.empty? # Can't check existence of bucket this way

data/scout-camp.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-camp 0.1.11 ruby lib
+# stub: scout-camp 0.1.13 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-camp".freeze
-  s.version = "0.1.11".freeze
+  s.version = "0.1.13".freeze
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Miguel Vazquez".freeze]
-  s.date = "2025-04-16"
+  s.date = "1980-01-02"
   s.description = "Functionalities to deploy and use scouts in remote servers like AWS".freeze
   s.email = "mikisvaz@gmail.com".freeze
   s.executables = ["scout-camp".freeze]
@@ -72,8 +72,12 @@ Gem::Specification.new do |s|
     "share/terraform/aws/efs_host/main.tf",
     "share/terraform/aws/efs_host/output.tf",
     "share/terraform/aws/efs_host/variables.tf",
+    "share/terraform/aws/event_bridge/data.tf",
+    "share/terraform/aws/event_bridge/main.tf",
+    "share/terraform/aws/event_bridge/variables.tf",
     "share/terraform/aws/fargate/locals.tf",
     "share/terraform/aws/fargate/main.tf",
+    "share/terraform/aws/fargate/output.tf",
     "share/terraform/aws/fargate/variables.tf",
     "share/terraform/aws/host/locals.tf",
     "share/terraform/aws/host/main.tf",
@@ -112,7 +116,7 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://github.com/mikisvaz/scout-camp".freeze
   s.licenses = ["MIT".freeze]
-  s.rubygems_version = "3.6.6".freeze
+  s.rubygems_version = "3.6.7".freeze
   s.summary = "Deploy you scouts".freeze
 
   s.specification_version = 4

data/scout_commands/terraform/lambda_task

@@ -14,6 +14,7 @@ $ #{$0} [<options>] <workflow> <task> [<other|->]*
 --prefix* Prefix, defaults to Scout
 --queue Queue job
 --clean Clean job
+--info Return job info
 --recursive_clean Clean job recursively
 EOF
 workflow, task_name = ARGV
@@ -27,9 +28,9 @@ if workflow.nil? && options[:help]
   exit 0
 end
 
-raise ParamterException, "No workflow specified" if workflow.nil?
+raise ParameterException, "No workflow specified" if workflow.nil?
 
-prefix, clean, recursive_clean, queue = IndiferentHash.process_options options, :prefix, :clean, :recursive_clean, :queue,
+prefix, clean, recursive_clean, queue, info = IndiferentHash.process_options options, :prefix, :clean, :recursive_clean, :queue, :info,
   prefix: "Scout"
 
 require 'aws-sdk-lambda'
@@ -116,6 +117,8 @@ res = case task_name
           exit 0
         end
 
+        payload["info"] = true if info
+
         aws_lambda(lambda_handler, payload)
       end
 

data/share/aws/lambda_function.rb

@@ -7,8 +7,13 @@ def lambda_handler(event:, context:)
   require 'scout/workflow'
   require 'scout/aws/s3'
 
-  workflow, task_name, jobname, inputs, clean, queue = IndiferentHash.process_options event,
-    :workflow, :task_name, :jobname, :inputs, :clean, :queue
+  TmpFile.tmpdir = Path.setup('/tmp')
+  Open.sensible_write_dir = Path.setup('/tmp/sensible_write')
+
+  Log.info "Payload: #{Log.fingerprint(event)}"
+
+  workflow, task_name, jobname, inputs, clean, queue, info = IndiferentHash.process_options event,
+    :workflow, :task_name, :jobname, :inputs, :clean, :queue, :info
 
   raise ParameterException, "No workflow specified" if workflow.nil?
 
@@ -21,6 +26,8 @@ def lambda_handler(event:, context:)
     raise ParameterException, "No task_name specified" if task_name.nil?
     return workflow.task_info(inputs["task_name"])
   else
+    Workflow.job_cache.clear
+
     job = workflow.job(task_name, jobname, inputs)
 
     case clean
@@ -30,7 +37,11 @@ def lambda_handler(event:, context:)
       job.recursive_clean
     end
 
-    if job.done?
+    if info
+      info = job.info.dup
+      info["path"] = job.path
+      info
+    elsif job.done?
       job.load
     elsif job.error?
       raise job.exception
@@ -41,7 +52,7 @@ def lambda_handler(event:, context:)
       }
     elsif queue
       save_inputs = Scout.var.queue[workflow.to_s][task_name][job.name].find :bucket
-      job.save_inputs(save_inputs)
+      job.save_input_bundle(save_inputs) unless save_inputs.exists?
       {
         statusCode: 202,
         body: job.path

data/share/terraform/aws/event_bridge/data.tf

@@ -0,0 +1,10 @@
+data "aws_vpc" "default" {
+  default = true
+}
+
+data "aws_subnets" "default" {
+  filter {
+    name   = "vpc-id"
+    values = [data.aws_vpc.default.id]
+  }
+}

data/share/terraform/aws/event_bridge/main.tf

@@ -0,0 +1,85 @@
+# IAM Role for EventBridge to trigger ECS task
+resource "aws_iam_role" "eventbridge_invoke_ecs" {
+  name = "eventbridge_invoke_ecs"
+
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17",
+    Statement = [
+      {
+        Effect = "Allow",
+        Principal = {
+          Service = "events.amazonaws.com"
+        },
+        Action = "sts:AssumeRole"
+      }
+    ]
+  })
+}
+
+# Permissions for EventBridge to run the task
+resource "aws_iam_role_policy" "ecs_task_invoke_policy" {
+  role = aws_iam_role.eventbridge_invoke_ecs.name
+
+  policy = jsonencode({
+    Version = "2012-10-17",
+    Statement = [
+      {
+        Effect = "Allow",
+        Action = [
+          "ecs:RunTask"
+        ],
+        Resource = "*"
+      },
+      {
+        Effect = "Allow",
+        Action = [
+          "iam:PassRole"
+        ],
+        Resource = "*"
+      }
+    ]
+  })
+}
+
+# EventBridge rule triggered on S3 put event
+resource "aws_cloudwatch_event_rule" "s3_put_event" {
+  name        = "s3-put-to-uploads"
+  description = "Triggers Fargate task on S3 file upload"
+  event_pattern = jsonencode({
+    source = ["aws.s3"],
+    "detail-type" = ["Object Created"],
+    detail = {
+      bucket = {
+        name = [var.bucket]
+      },
+      object = {
+        key = [{
+          prefix = var.directory
+        }]
+      }
+    }
+  })
+}
+
+# Target: ECS Fargate task
+resource "aws_cloudwatch_event_target" "run_fargate" {
+  rule      = aws_cloudwatch_event_rule.s3_put_event.name
+  arn       = var.cluster_arn # Replace with your ECS Cluster ARN
+  role_arn  = aws_iam_role.eventbridge_invoke_ecs.arn
+
+  ecs_target {
+    task_definition_arn = var.task_arn
+    launch_type         = "FARGATE"
+    network_configuration {
+      assign_public_ip = true
+      subnets          = data.aws_subnets.default.ids
+    }
+  }
+}
+
+# Allow S3 to send events to EventBridge
+#resource "aws_s3_bucket_notification" "s3_event" {
+#  bucket = var.bucket_name
+#
+#  eventbridge = true
+#}

data/share/terraform/aws/event_bridge/variables.tf

@@ -0,0 +1,19 @@
+# Replace with your actual bucket
+variable "bucket" {
+  type = string
+}
+
+variable "directory" {
+  type = string
+}
+
+variable "task_arn" {
+  type = string
+}
+
+variable "cluster_arn" {
+  type = string
+}
+
+
+

data/share/terraform/aws/fargate/output.tf

@@ -0,0 +1,10 @@
+output "task_arn" {
+  description = "Task arn"
+  value = aws_ecs_task_definition.this.arn
+}
+
+output "cluster_arn" {
+  description = "Task arn"
+  value = aws_ecs_cluster.this.arn
+}
+

data/test/scout/aws/test_s3.rb

@@ -21,6 +21,15 @@ class TestS3 < Test::Unit::TestCase
     Open.rm uri
   end
 
+  def test_touch_size
+    uri = "s3://herlab/tmp/foo.txt"
+
+    Open.rm uri
+    Open.touch uri
+    assert_equal 0, Open.size(uri)
+    Open.rm uri
+  end
+
   def test_write_io
     uri = "s3://herlab/tmp/foo.txt"
 

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: scout-camp
 version: !ruby/object:Gem::Version
-  version: 0.1.11
+  version: 0.1.13
 platform: ruby
 authors:
 - Miguel Vazquez
 bindir: bin
 cert_chain: []
-date: 2025-04-16 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: scout-essentials
@@ -84,8 +84,12 @@ files:
 - share/terraform/aws/efs_host/main.tf
 - share/terraform/aws/efs_host/output.tf
 - share/terraform/aws/efs_host/variables.tf
+- share/terraform/aws/event_bridge/data.tf
+- share/terraform/aws/event_bridge/main.tf
+- share/terraform/aws/event_bridge/variables.tf
 - share/terraform/aws/fargate/locals.tf
 - share/terraform/aws/fargate/main.tf
+- share/terraform/aws/fargate/output.tf
 - share/terraform/aws/fargate/variables.tf
 - share/terraform/aws/host/locals.tf
 - share/terraform/aws/host/main.tf
@@ -139,7 +143,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.6
+rubygems_version: 3.6.7
 specification_version: 4
 summary: Deploy you scouts
 test_files: []