scout-camp 0.1.1

data/share/terraform/aws/cluster/variables.tf

@@ -0,0 +1,49 @@
+variable "cidr_block_base" {
+  description = "CIDR block base IP to use in the vpc (e.g. 10.0.0.0)"
+  type = string
+  default = "10.0.0.0"
+}
+
+variable "cidr_block_mask" {
+  description = "CIDR block mask to use in the vpc (e.g. 16)"
+  type = string
+  default = "16"
+}
+
+variable "map_public_ip_on_launch" {
+  description = "Map the publich ip on launch"
+  type = bool
+  default = true
+}
+
+variable "availability_zone" {
+  description = "Availability zone for cluster"
+  type = string
+  default = null
+}
+
+# NAMETAGS
+
+variable "cidr_nametag" {
+  description = "Name tag to assign the aws CIDR"
+  type = string
+  default = "one-provision-aws_CIDR"
+}
+
+variable "subnet_nametag" {
+  description = "Name tag to assign the aws subnet"
+  type = string
+  default = "one-provision-aws_subnet"
+}
+
+variable "gateway_nametag" {
+  description = "Name tag to assign the aws gateway"
+  type = string
+  default = "one-provision-aws_gateway"
+}
+
+variable "security_group_nametag" {
+  description = "Name tag to assign the aws security group"
+  type = string
+  default = "one-provision-aws_security_group"
+}

data/share/terraform/aws/host/locals.tf

@@ -0,0 +1,15 @@
+locals {
+
+  ssh_key_list = var.ssh_key != null ? [var.ssh_key] : (var.ssh_keys == null ? [] : var.ssh_keys)
+
+  ssh_key_txt = "- ${join("\n- ",local.ssh_key_list)} "
+
+  cloud_config =<<EOF
+#cloud-config
+ssh_authorized_keys:
+${local.ssh_key_txt}
+EOF
+
+  user_data     = var.user_data != null ?  var.user_data : base64encode(local.cloud_config)
+}
+

data/share/terraform/aws/host/main.tf

@@ -0,0 +1,22 @@
+resource "aws_instance" "this" {
+  ami           = var.ami
+  instance_type = var.instance_type
+
+  private_ip    = var.private_ip
+  subnet_id     = var.subnet_id
+
+  vpc_security_group_ids = var.vpc_security_group_ids
+
+  user_data     = local.user_data
+
+  availability_zone      = var.availability_zone
+
+  root_block_device {
+      volume_size = var.volume_size
+  }
+
+  tags = {
+    Name = var.host_nametag
+  }
+}
+

data/share/terraform/aws/host/output.tf

@@ -0,0 +1,9 @@
+output "aws_instance_id" {
+  description = "Code that identifies the AWS instance"
+  value = aws_instance.this.id
+}
+
+output "aws_instance_ip" {
+  description = "Public IP address of the AWS instance"
+  value = aws_instance.this.public_ip
+}

data/share/terraform/aws/host/variables.tf

@@ -0,0 +1,67 @@
+
+variable "ami" {
+  description = "AMI id for host"
+  type = string
+}
+
+variable "instance_type" {
+  description = "Type of AWS instance"
+  type = string
+  default = "t2.micro"
+}
+
+variable "volume_size" {
+  description = "Size of volumes"
+  type = number
+  default = 512
+}
+
+variable "private_ip" {
+  description = "Host private IP"
+  type = string
+  default = null
+}
+
+variable "user_data" {
+  description = "User data for host"
+  type = string
+  default = null
+}
+
+variable "ssh_key" {
+  description = "SSH key"
+  type = string
+  default = null
+}
+
+variable "ssh_keys" {
+  description = "List of SSH key"
+  type = list(string)
+  default = null
+}
+
+variable "availability_zone" {
+  description = "Availability zone for host"
+  type = string
+  default = null
+}
+
+
+variable "subnet_id" {
+  description = "Cluster subnet id"
+  type = string
+  default = null
+}
+
+variable "vpc_security_group_ids" {
+  description = "List of security group ids"
+  type = list(string)
+  default = null
+}
+
+variable "host_nametag" {
+  description = "Name tag to assign the AWS instance"
+  type = string
+  default = "one-provision-aws_instance"
+}
+

data/share/terraform/aws/lambda/main.tf

@@ -0,0 +1,40 @@
+provider "aws" {
+  region = "eu-west-2"
+}
+
+resource "aws_lambda_function" "this" {
+  function_name = var.function_name
+  role          = aws_iam_role.lambda_role.arn
+  handler       = "lambda_function.lambda_handler"
+  runtime       = var.runtime
+  filename      = var.filename
+  source_code_hash = filebase64sha256(var.filename)
+  timeout       = var.timeout
+
+  environment {
+    variables = var.environment_variables
+  }
+}
+
+resource "aws_iam_role" "lambda_role" {
+  name = "lambda_execution_role"
+  
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Action = "sts:AssumeRole"
+        Effect = "Allow"
+        Principal = {
+          Service = "lambda.amazonaws.com"
+        }
+      }
+    ]
+  })
+}
+
+resource "aws_iam_policy_attachment" "lambda_basic" {
+  name       = "lambda_basic"
+  roles      = [aws_iam_role.lambda_role.name]
+  policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
+}

data/share/terraform/aws/lambda/variables.tf

@@ -0,0 +1,23 @@
+variable "function_name" {
+  description = "Lambda function name"
+  type = string
+}
+variable "runtime" {
+  description = "Ruby runtime"
+  type = string
+  default = "ruby3.2"
+}
+variable "filename" {
+  description = "ZIP filename with lambda handler"
+  type = string
+}
+variable "timeout" {
+  description = "Timeout for call"
+  type = number
+  default = 30
+}
+variable "environment_variables" {
+  type        = map(string)
+  description = "A map of environment variables to pass to the resource"
+  default     = {}
+}

data/share/terraform/aws/provider/data.tf

@@ -0,0 +1,35 @@
+data "aws_availability_zones" "available" {}
+
+data "aws_ec2_instance_type_offerings" "available" {
+
+    for_each=toset(data.aws_availability_zones.available.names)
+
+    filter {
+        name   = "instance-type"
+        values = ["c5.metal"]
+    }
+
+    filter {
+        name   = "location"
+        values = [each.key]
+    }
+
+    location_type = "availability-zone"
+}
+
+data "aws_ami" "ubuntu2004" {
+
+    most_recent = true
+
+    filter {
+        name   = "name"
+        values = ["ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*"]
+    }
+
+    filter {
+        name = "virtualization-type"
+        values = ["hvm"]
+    }
+
+    owners = ["099720109477"]
+}

data/share/terraform/aws/provider/output.tf

@@ -0,0 +1,16 @@
+output "available_aws_offerings" {
+  value = data.aws_ec2_instance_type_offerings.available
+}
+
+output "default_ami" {
+  value = data.aws_ami.ubuntu2004.id
+}
+
+output "my_zones" {
+  value = keys({ for az, details in data.aws_ec2_instance_type_offerings.available :
+                      az => details.instance_types if length(details.instance_types) != 0 })
+}
+
+output "ipam" {
+  value = "aws"
+}

data/test/scout/aws/test_s3.rb

@@ -0,0 +1,82 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/workflow'
+class TestS3 < Test::Unit::TestCase
+  def test_list
+    file = "s3://herlab/tmp/foo.txt"
+
+    Open::S3.write file, "TEST"
+  
+    dir = "s3://herlab/tmp"
+
+    assert_include Open::S3.glob(dir, "**/*"), file
+  end
+
+  def test_write
+    uri = "s3://herlab/tmp/foo.txt"
+
+    Open.write uri, "TEST"
+    assert_equal "TEST", Open.read(uri)
+    Open.rm uri
+  end
+
+  def test_write_io
+    uri = "s3://herlab/tmp/foo.txt"
+
+    Open::S3.write uri, "TEST"
+    io = Open::S3.get_stream uri
+    assert_equal "TEST", io.read
+    Open::S3.rm uri
+  end
+
+  def test_write_block
+    uri = "s3://herlab/tmp/foo.txt"
+
+    Open::S3.write uri do |sin|
+      sin.write "TEST"
+    end
+    io = Open::S3.get_stream uri
+    assert_equal "TEST", io.read
+    Open::S3.rm uri
+  end
+
+  def test_step
+    s = Step.new "s3://herlab/var/jobs/Baking/bake_muffin_tray/Default"
+    assert_include s.load, "baking"
+  end
+
+  def test_tmpfile
+    require 'rbbt-util'
+
+    TmpFile.tmpdir = Path.setup("s3://herlab/tmp")
+
+    TmpFile.with_path "TEST" do |file|
+      assert Open::S3.is_s3? file
+      assert_equal "TEST", file.read
+    end
+  end
+
+  def test_workflow
+    m = Module.new do
+      extend Workflow
+      name = "TestWF"
+
+      task :step1 => :string do
+        file('foo').write('bar')
+        "Step 1"
+      end
+
+      dep :step1
+      task :step2 => :string do
+        step(:step1).file('foo').read
+      end
+    end
+
+    m.directory = Path.setup("s3://herlab/var/jobs/TestWF")
+
+    m.job(:step2).run
+    assert_include m.job(:step2).step(:step1).files, 'foo'
+  end
+end
+

data/test/scout/offsite/test_ssh.rb

@@ -0,0 +1,15 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestSSH < Test::Unit::TestCase
+  def test_marshal
+    return unless SSHLine.reach?
+
+    assert TrueClass === SSHLine.scout(:default, 'true')
+  end
+
+  def test_localhost
+    assert SSHLine.scout('localhost', 'true')
+  end
+end
+

data/test/scout/offsite/test_step.rb

@@ -0,0 +1,33 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestOffsiteStep < Test::Unit::TestCase
+  def __test_offsite_task
+    workflow_code =<<-EOF
+module TestWF
+  extend Workflow
+
+  input :string, :string, "String", "string"
+  task :string => :string do |string| string end
+end
+
+TestWF.directory = Path.setup("#{tmpdir.offsite.TestWF}")
+    EOF
+
+    sss 0
+    TmpFile.with_file workflow_code, :extension => 'rb' do |wffile|
+      wf = Workflow.require_workflow wffile
+
+      job = wf.job(:string)
+
+      off = OffsiteStep.setup job, server: 'localhost', workflow_name: wffile
+
+      refute off.done?
+      assert_equal 'string', off.run
+
+      assert off.done?
+      assert_equal 'string', off.run
+    end
+  end
+end
+

data/test/scout/offsite/test_sync.rb

@@ -0,0 +1,36 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestSync < Test::Unit::TestCase
+  def test_sync
+    TmpFile.with_path do |tmpdir|
+      tmpdir = Scout.tmp.tmpdir_sync
+      tmpdir.dir1.foo.write("FOO")
+      tmpdir.dir1.bar.write("BAR")
+
+      TmpFile.with_path do |tmpdir2|
+        Misc.in_dir tmpdir2 do
+          SSHLine.sync([tmpdir.dir1], map: :current)
+
+          assert tmpdir2.glob("**/*").select{|f| f.include?('foo') }.any?
+        end
+      end
+    end
+  end
+
+  def test_sync_dir_map
+    TmpFile.with_path do |tmpdir|
+      tmpdir = Scout.tmp.tmpdir_sync
+      tmpdir.dir1.foo.write("FOO")
+      tmpdir.dir1.bar.write("BAR")
+
+      TmpFile.with_path do |tmpdir2|
+        SSHLine.sync([tmpdir.dir1], map: tmpdir2)
+        Misc.in_dir tmpdir2 do
+          assert tmpdir2.glob("**/*").select{|f| f.include?('foo') }.any?
+        end
+      end
+    end
+  end
+end
+