scout-camp 0.1.6 → 0.1.10

data/share/aws/lambda_function.rb

@@ -7,21 +7,48 @@ def lambda_handler(event:, context:)
   require 'scout/workflow'
   require 'scout/aws/s3'
 
-  workflow, task_name, jobname, inputs = IndiferentHash.process_options event, 
-  :workflow, :task_name, :jobname, :inputs
-  
-  raise ParamterException, "No workflow specified" if workflow.nil?
-  
+  workflow, task_name, jobname, inputs, clean, queue = IndiferentHash.process_options event,
+    :workflow, :task_name, :jobname, :inputs, :clean, :queue
+
+  raise ParameterException, "No workflow specified" if workflow.nil?
+
   workflow = Workflow.require_workflow workflow
 
   case task_name
   when nil
     return {tasks: workflow.tasks.keys, documentation: workflow.documentation}
   when "info"
-    raise ParamterException, "No task_name specified" if task_name.nil?
+    raise ParameterException, "No task_name specified" if task_name.nil?
     return workflow.task_info(inputs["task_name"])
   else
     job = workflow.job(task_name, jobname, inputs)
-    job.run
+
+    case clean
+    when true, 'true'
+      job.clean
+    when 'recursive'
+      job.recursive_clean
+    end
+
+    if job.done?
+      job.load
+    elsif job.error?
+      raise job.exception
+    elsif job.started?
+      {
+        statusCode: 202,
+        body: job.path
+      }
+    elsif queue
+      save_inputs = Scout.var.queue[workflow.to_s][task_name][job.name].find
+      job.save_inputs(save_inputs)
+      {
+        statusCode: 202,
+        body: job.path
+      }
+    else
+      job.produce
+      job.load
+    end
   end
 end

data/share/terraform/aws/container_lambda/data.tf

@@ -0,0 +1,15 @@
+data "aws_vpc" "default" {
+  default = true
+}
+
+data "aws_subnets" "default_vpc_subnets" {
+  filter {
+    name   = "vpc-id"
+    values = [data.aws_vpc.default.id]
+  }
+
+  filter {
+    name   = "default-for-az"
+    values = ["true"]
+  }
+}

data/share/terraform/aws/container_lambda/locals.tf

@@ -0,0 +1,8 @@
+locals {
+  security_group_ids = [
+    for sg_key in var.sg_keys :
+    lookup(var.network.outputs, sg_key, null)
+  ]
+
+  efs_id = lookup(var.efs.outputs, "aws_efs_id", null)
+}

data/share/terraform/aws/container_lambda/main.tf

@@ -0,0 +1,47 @@
+resource "aws_efs_access_point" "lambda_ap" {
+  file_system_id = local.efs_id
+
+  posix_user {
+    uid = 1000
+    gid = 1000
+  }
+
+  root_directory {
+    path = var.mount_point
+    creation_info {
+      owner_uid   = 1000
+      owner_gid   = 1000
+      permissions = "755"
+    }
+  }
+}
+
+resource "aws_lambda_function" "this" {
+  function_name = var.function_name
+  package_type  = "Image"
+
+  image_uri = var.image
+
+  role = var.role_arn
+
+  timeout     = var.timeout
+  memory_size = var.memory
+
+  environment {
+    variables = var.environment_variables
+  }
+
+  vpc_config {
+    subnet_ids         = data.aws_subnets.default_vpc_subnets.ids
+    security_group_ids = local.security_group_ids
+  }
+
+  file_system_config {
+    arn              = aws_efs_access_point.lambda_ap.arn
+    local_mount_path = "/mnt/efs"
+  }
+
+  image_config {
+    command = ["app.handler"]
+  }
+}

data/share/terraform/aws/container_lambda/variables.tf

@@ -0,0 +1,44 @@
+variable "function_name" {
+  description = "Lambda function name"
+  type = string
+}
+variable "timeout" {
+  description = "Timeout for call"
+  type = number
+  default = 30
+}
+variable "environment_variables" {
+  type        = map(string)
+  description = "A map of environment variables to pass to the resource"
+  default     = {}
+}
+variable "role_arn" {
+}
+variable "image" {
+}
+variable "memory" {
+  type        = number
+  description = "The memory (MiB) for the task"
+  default     = 512
+}
+
+variable "network" {
+  description = "Name of the remote state block to use for the network"
+}
+
+variable "efs" {
+  description = "Name of the remote state block to use for the EFS"
+}
+
+variable "sg_keys" {
+  description = "List of output names in the remote state representing security group IDs"
+  type        = list(string)
+  default     = ["aws_network_efs_sg_id", "aws_network_ssh_sg_id"]
+}
+
+variable "mount_point" {
+  description = "Where to mount the efs drive"
+  type = string
+  default = "/mnt/efs"
+}
+

data/share/terraform/aws/efs/data.tf

@@ -0,0 +1,12 @@
+# Get default VPC (optional if you want to restrict to default VPC)
+data "aws_vpc" "default" {
+  default = true
+}
+
+# Get all subnets in the region (filtered to default VPC if needed)
+data "aws_subnets" "all" {
+  filter {
+    name   = "vpc-id"
+    values = [data.aws_vpc.default.id]
+  }
+}

data/share/terraform/aws/efs/locals.tf

@@ -0,0 +1,6 @@
+locals {
+  security_group_ids = [
+    for sg_key in var.sg_keys :
+    lookup(var.remote.outputs, sg_key, null)
+  ]
+}

data/share/terraform/aws/efs/main.tf

@@ -0,0 +1,14 @@
+resource "aws_efs_file_system" "this" {
+  creation_token = "herlab-efs"
+  tags = {
+    Name = "HERLab main EFS"
+  }
+}
+
+resource "aws_efs_mount_target" "this" {
+  for_each = toset(data.aws_subnets.all.ids)
+
+  file_system_id  = aws_efs_file_system.this.id
+  subnet_id       = each.value
+  security_groups = local.security_group_ids
+}

data/share/terraform/aws/efs/output.tf

@@ -0,0 +1,3 @@
+output "id" {
+  value = aws_efs_file_system.this.id
+}

data/share/terraform/aws/efs/variables.tf

@@ -0,0 +1,9 @@
+variable "remote" {
+  description = "Name of the remote state block to use"
+}
+
+variable "sg_keys" {
+  description = "List of output names in the remote state representing security group IDs"
+  type        = list(string)
+  default     = ["aws_network_efs_sg_id"]
+}

data/share/terraform/aws/efs_host/data.tf

@@ -0,0 +1,11 @@
+data "aws_ami" "amazon_linux_2" {
+  most_recent = true
+  owners      = ["amazon"]
+
+  # Filter for Amazon Linux 2 AMIs
+  filter {
+    name   = "name"
+    values = ["amzn2-ami-hvm-*-x86_64-gp2"]
+  }
+}
+

data/share/terraform/aws/efs_host/locals.tf

@@ -0,0 +1,8 @@
+locals {
+  security_group_ids = [
+    for sg_key in var.sg_keys :
+    lookup(var.network.outputs, sg_key, null)
+  ]
+
+  efs_id = lookup(var.efs.outputs, "aws_efs_id", null)
+}

data/share/terraform/aws/efs_host/main.tf

@@ -0,0 +1,32 @@
+resource "aws_key_pair" "this" {
+  key_name   = "my-key"
+  public_key = file("~/.ssh/id_rsa.pub") # Adjust if your key is elsewhere
+}
+
+resource "aws_instance" "this" {
+  ami           = data.aws_ami.amazon_linux_2.id
+  instance_type = "t2.micro"
+  iam_instance_profile = var.policies.outputs.ec2_host_profile_id
+
+  key_name      = aws_key_pair.this.key_name
+
+  tags = {
+    Name = "EFS-Service"
+  }
+
+  # Open port 22 for SSH
+  vpc_security_group_ids = local.security_group_ids
+
+  user_data = <<-EOF
+              #cloud-config
+              package_update: true
+              package_upgrade: true
+              packages:
+                - amazon-efs-utils
+              runcmd:
+                - mkdir -p ${var.mount_point}
+                - mount -t efs -o tls ${local.efs_id}:/ ${var.mount_point}
+                - echo "${local.efs_id}:/ ${var.mount_point} efs defaults,_netdev 0 0" >> /etc/fstab
+EOF
+}
+

data/share/terraform/aws/efs_host/output.tf

@@ -0,0 +1,3 @@
+output "public_ip" {
+  value = aws_instance.this.public_ip
+}

data/share/terraform/aws/efs_host/variables.tf

@@ -0,0 +1,28 @@
+variable "network" {
+  description = "Name of the remote state block to use for the network"
+}
+
+variable "efs" {
+  description = "Name of the remote state block to use for the EFS"
+}
+
+variable "policies" {
+  description = "Name of the remote state block to use for the policies"
+}
+
+variable "sg_keys" {
+  description = "List of output names in the remote state representing security group IDs"
+  type        = list(string)
+  default     = ["aws_network_efs_sg_id", "aws_network_ssh_sg_id"]
+}
+variable "mount_point" {
+  description = "Where to mount the efs drive"
+  type = string
+  default = "/mnt/efs"
+}
+
+#variable "iam_instance_profile" {
+#  description = "Instance profile"
+#  type = string
+#  default = null
+#}

data/share/terraform/aws/fargate/locals.tf

@@ -0,0 +1,8 @@
+locals {
+  security_group_ids = [
+    for sg_key in var.sg_keys :
+    lookup(var.network.outputs, sg_key, null)
+  ]
+
+  efs_id = lookup(var.efs.outputs, "aws_efs_id", null)
+}

data/share/terraform/aws/fargate/main.tf

@@ -0,0 +1,41 @@
+resource "aws_ecs_task_definition" "this" {
+  family                   = var.task_family
+  requires_compatibilities = ["FARGATE"]
+  network_mode             = "awsvpc"
+  cpu                      = var.cpu
+  memory                   = var.memory
+  execution_role_arn       = var.policies.outputs.fargate_execution_role_arn
+  task_role_arn            = var.policies.outputs.fargate_task_role_arn
+
+  container_definitions = jsonencode([
+    {
+      name      = var.container_name
+      image     = var.image
+      user     = var.user
+      essential = true
+      portMappings = var.port_mappings
+      //entryPoint = var.entry_point
+      command = var.command
+      environment = var.environment
+
+      mountPoints = [
+        {
+          sourceVolume  = "efs-volume"
+          containerPath = var.mount_point
+        }
+      ]
+    }
+  ])
+
+  volume {
+    name = "efs-volume"
+    efs_volume_configuration {
+      file_system_id = local.efs_id
+      root_directory = "/"
+    }
+  }
+}
+
+resource "aws_ecs_cluster" "this" {
+  name = "${var.task_family}_cluster"
+}

data/share/terraform/aws/fargate/variables.tf

@@ -0,0 +1,84 @@
+variable "network" {
+  description = "Name of the remote state block to use for the network"
+}
+
+variable "efs" {
+  description = "Name of the remote state block to use for the EFS"
+}
+
+variable "policies" {
+  description = "Name of the remote state block to use for the policies"
+}
+
+variable "sg_keys" {
+  description = "List of output names in the remote state representing security group IDs"
+  type        = list(string)
+  default     = ["aws_network_efs_sg_id", "aws_network_ssh_sg_id"]
+}
+
+variable "mount_point" {
+  description = "Where to mount the efs drive"
+  type = string
+  default = "/mnt/efs"
+}
+
+variable "task_family" {
+  type        = string
+  description = "The family name of the ECS task definition"
+}
+
+variable "cpu" {
+  type        = number
+  description = "The CPU units for the task"
+  default     = 256
+}
+
+variable "memory" {
+  type        = number
+  description = "The memory (MiB) for the task"
+  default     = 512
+}
+
+variable "container_name" {
+  type        = string
+  description = "Name of the container"
+  default     = "app"
+}
+
+variable "image" {
+  type        = string
+  description = "Docker image URL for the container"
+}
+
+variable "user" {
+  description = "User to use"
+  type        = string
+  default = null
+}
+
+variable "port_mappings" {
+  type = list(object({
+    containerPort = number
+    hostPort      = number
+    protocol      = string
+  }))
+  description = "List of port mappings for the container"
+  default     = []
+}
+
+variable "command" {
+  type        = list(string)
+  description = "Command to run"
+}
+
+variable "entry_point" {
+  type        = list(string)
+  description = "Container entry point"
+  default     = ["bash"]
+}
+
+variable "environment" {
+  type        = map(string)
+  description = "A map of environment variables to pass to the resource"
+  default     = null
+}

data/share/terraform/aws/iam_instance_profile/main.tf

@@ -0,0 +1,5 @@
+resource "aws_iam_instance_profile" "this" {
+  name = var.profile_name
+  role = var.role
+}
+

data/share/terraform/aws/iam_instance_profile/output.tf

@@ -0,0 +1,15 @@
+output "arn" {
+  description = "Instance profile arn"
+  value = aws_iam_instance_profile.this.arn
+}
+
+output "profile_name" {
+  description = "Instance profile name"
+  value = aws_iam_instance_profile.this.name
+}
+
+output "id" {
+  description = "Instance profile id"
+  value = aws_iam_instance_profile.this.id
+}
+

data/share/terraform/aws/iam_instance_profile/variables.tf

@@ -0,0 +1,9 @@
+variable "role" {
+  description = "Role to assign to the instance profile"
+  type = string
+}
+
+variable "profile_name" {
+  description = "Profile name"
+  type = string
+}

data/share/terraform/aws/lambda/main.tf

@@ -5,7 +5,7 @@ resource "aws_lambda_function" "this" {
   filename      = var.filename
   source_code_hash = filebase64sha256(var.filename)
   timeout       = var.timeout
-  role          = var.role
+  role          = var.policies.outputs.lambda_execution_role_arn
 
   environment {
     variables = var.environment_variables

data/share/terraform/aws/lambda/variables.tf

@@ -5,7 +5,7 @@ variable "function_name" {
 variable "runtime" {
   description = "Ruby runtime"
   type = string
-  default = "ruby3.2"
+  default = "ruby3.3"
 }
 variable "filename" {
   description = "ZIP filename with lambda handler"
@@ -21,7 +21,6 @@ variable "environment_variables" {
   description = "A map of environment variables to pass to the resource"
   default     = {}
 }
-variable "role" {
-  description = "Role to assume"
-  type = string
+variable "policies" {
+  description = "Name of the remote state block to use for the policies"
 }

data/share/terraform/aws/network/data.tf

@@ -0,0 +1,15 @@
+data "aws_vpc" "default" {
+  filter {
+    name   = "is-default"
+    values = ["true"]
+  }
+}
+
+# Get all subnets in the default VPC
+data "aws_subnets" "default" {
+  filter {
+    name   = "vpc-id"
+    values = [data.aws_vpc.default.id]
+  }
+}
+

data/share/terraform/aws/network/main.tf

@@ -0,0 +1,41 @@
+resource "aws_security_group" "efs" {
+  name        = "efs-sg"
+  description = "Allow NFS"
+
+  ingress {
+    from_port   = 2049
+    to_port     = 2049
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  tags = {
+    Name = "efs-sg"
+  }
+}
+
+resource "aws_security_group" "ssh" {
+  name        = "allow_ssh"
+  description = "Allow SSH inbound traffic"
+
+  ingress {
+    from_port   = 22
+    to_port     = 22
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]  # WARNING: open to the world. Limit this for production.
+  }
+
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}

data/share/terraform/aws/network/output.tf

@@ -0,0 +1,7 @@
+output "efs_sg_id" {
+  value = aws_security_group.efs.id
+}
+
+output "ssh_sg_id" {
+  value = aws_security_group.ssh.id
+}

data/share/terraform/aws/policy/main.tf

@@ -0,0 +1,8 @@
+resource "aws_iam_policy" "this" {
+  name = var.policy_name
+  policy = jsonencode({
+    Version = "2012-10-17",
+    Statement = var.statement
+  })
+}
+

data/share/terraform/aws/policy/output.tf

@@ -0,0 +1,15 @@
+output "arn" {
+  description = "Policy arn"
+  value = aws_iam_policy.this.arn
+}
+
+output "name" {
+  description = "Policy name"
+  value = aws_iam_policy.this.name
+}
+
+output "id" {
+  description = "Policy id"
+  value = aws_iam_policy.this.id
+}
+

data/share/terraform/aws/policy/variables.tf

@@ -0,0 +1,12 @@
+variable "policy_name" {
+  type    = string
+}
+
+variable "statement" {
+  type    = list(object({
+    Action   = any
+    Effect   = string
+    Resource = any
+  }))
+}
+

data/share/terraform/aws/role/main.tf

@@ -5,7 +5,7 @@ resource "aws_iam_role" "this" {
     Version = "2012-10-17"
     Statement = [
       {
-        Action = "sts:AssumeRole"
+        Action = var.action
         Effect = "Allow"
         Principal = var.principal  
       }

data/share/terraform/aws/role/output.tf

@@ -3,7 +3,7 @@ output "arn" {
   value = aws_iam_role.this.arn
 }
 
-output "name" {
+output "role_name" {
   description = "Role name"
   value = aws_iam_role.this.name
 }

data/share/terraform/aws/role/variables.tf

@@ -8,4 +8,7 @@ variable "principal" {
   type = map(any)
 }
 
-
+variable "action" {
+  description = "Action of the role"
+  default = "sts:AssumeRole"
+}

data/share/terraform/aws/role_policy/main.tf

@@ -0,0 +1,9 @@
+resource "aws_iam_role_policy" "this" {
+  name = var.policy_name
+  role = var.role
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = var.statement
+  })
+}