scout-camp 0.1.1

data/lib/scout/terraform_dsl.rb

@@ -0,0 +1,317 @@
+require_relative 'terraform_dsl/util'
+require_relative 'terraform_dsl/deployment'
+require 'scout-gear'
+require 'json'
+
+# Objects of this class contain the elements that will form
+# a terraform deployment configuration
+class TerraformDSL
+
+  attr_accessor :elements, :modules, :name, :processed_elements, :processed_custom_files
+
+  # Module objects hold the identity of a module and
+  # its :name, and can be used to create
+  # references to its outputs to pass as inputs for
+  # another module
+  class Module
+
+    attr_accessor :name, :type, :deployment
+
+    # Output objects are references to a Module output
+    # that can be used as parameters for other modules
+    # inputs
+    class Output
+
+      attr_accessor :module, :name
+
+      # Create an Output object
+      #
+      # @param mod [String] module template name
+      # @param name [String] instance name
+      def initialize(mod, name)
+        @module = mod
+        @name = name
+      end
+
+      # Callback to produce the json content when we serialize
+      # variable values. It becomes a reference to a Module
+      # output variable
+      #
+      # @param args [Array] Extra arguments to to_json, not used
+      # @return [String] A reference of an output for terraform 
+      #         (e.g.module.<modulename>.<name>)
+      def to_json(*_args)
+        ['module', @module, @name].join('.')
+      end
+
+    end
+
+    # Create a new Module
+    #
+    # @param name [String] instance name
+    # @param type [String] module template name
+    def initialize(name, type, deployment)
+      @name = name
+      @type = type
+      @deployment = deployment
+    end
+
+    # Construct output variable references to use on templates. They
+    # will be serialized when used in templates
+    #
+    # @param output [String] the name of the output
+    # @return [Output] An output variable for Terraform templates
+    def method_missing(output)
+      Output.new(@name, output)
+    end
+
+    # Any missing method call can be the name of an output
+    def respond_to_missing?(_method_name, _include_private = false)
+      true
+    end
+
+    # Callback to produce the json content when we serialize
+    # variable values. It becomes a reference to a Module itself,
+    # and can be used in depends_on statements
+    #
+    # @param args [Array] Extra arguments to to_json, not used
+    # @return [String] A reference to a module in Terraform format
+    #         (e.g. module.<modulename>)
+    def to_json(*_args)
+      ['module', @name].join('.')
+    end
+
+  end
+
+  MODULES_DIR = Scout.share.terraform
+  ANSIBLE_DIR = Scout.share.ansible
+  WORK_DIR = Scout.var.terraform
+
+  # Create a new terraform deployment configuration
+  #
+  # @param modules [String] directory containing module templates
+  def initialize(modules = MODULES_DIR)
+    @modules = modules
+    @elements = []
+    @custom_files = []
+    @variables = {}
+    @element_files = []
+  end
+
+  # Add a new module instance
+  #
+  # @param provider [String] first level of template organization subdirectory
+  # @param module_name [String] subdirectory containing the module to use
+  # @param variables [Hash] values for variables in the module template, and
+  #   :name to name the module instance, and :outputs to define the module
+  #   variables that will become deployment outputs
+  # @return [Module] A module object used as a reference in Terraform templates
+  def add(provider = nil, module_name = nil, variables = {})
+    variables[:name] ||= variables["name"] ||= [provider, module_name].join('_')
+    module_directory = @modules[provider][module_name]
+    @elements << [provider, module_name, module_directory, variables]
+    @variables.merge!(variables)
+    Module.new(variables[:name], module_name, self)
+  end
+
+  # Terraform text that describes variables passed to a given module instance
+  #
+  # @param variables [Hash] module variables and their values, :name and
+  #   :outputs are ignored as they are not module variables themselves
+  #
+  # @return [String] text to include inside the terraform module definition
+  #   containing the variable assignments
+  def variable_block(variables)
+    variables.each_with_object([]) do |p, acc|
+      name, value = p
+      next acc if name.to_s == 'name'
+      next acc if name.to_s == 'outputs'
+
+      if value.is_a?(String) && (m = value.match(/^module\.(.*)\.(.*)/))
+        value = Module::Output.new m[1], m[2]
+      end
+
+      acc << "  #{name} = #{value.to_json}"
+    end * "\n"
+  end
+
+  # Populate a directory with the terraform templates corresponding to the
+  # defined elements
+  #
+  # @param dir [String] directory from which to manage the deployment
+  def main(dir)
+    @elements.each do |info|
+      _provider, _module_name, template, variables = info
+
+      template = template.find
+      # Add an additional / to mark the base_path of the module directory and
+      # allow modules to reference other modules relatively
+      template = template.split('/').tap {|l| l[-2] = '/'+l[-2] } * '/'
+
+      name = variables[:name]
+
+      text =<<~EOF
+          module "#{name}" {
+            source = "#{template}"
+          #{variable_block(variables)}
+          }
+      EOF
+
+      element_file = [_module_name, name.to_s.sub(/_#{_module_name}$/,'')] * "."
+
+      # rubocop: disable Layout/LineLength
+      raise Deployment::TerraformException,
+        "Warning: element file '#{element_file}' already exists, consider renaming it by using the parameter ':name'" if @element_files.include?(element_file)
+      # rubocop: enable Layout/LineLength
+
+      @element_files << element_file
+
+      Open.write(dir[element_file + '.tf'], text)
+    end
+  end
+
+  # Add a terraform file with custom content. Used only
+  # to support defining non Hashicorp provider configuration
+  #
+  # @param file [String] name of the file
+  # @param text [String] content of the file
+  def custom(file, text)
+    @custom_files << [file, text]
+    nil
+  end
+
+  # Add a provider template file without using modules.
+  # Defining providers in modules is problematic when providers
+  # are not managed by Hashicorp. Hopefully we can
+  # find a fix for this soon.
+  #
+  # @param name [String] name of the provider
+  # @param variables [Hash] variables for the provider:
+  #   :source & :version
+  # @return [nil, Module] If a Module is found at <modules_dir>/<name>/provider 
+  #         it returns it
+  def provider(name, variables = {})
+    variables = variables.dup
+
+    if Open.exist?(@modules[name].provider)
+      provider = add name, :provider
+    else
+      provider = nil
+    end
+
+    source = variables.delete :source
+    version = variables.delete :version
+
+    text = ''
+
+    if source
+      if version
+        text +=<<~EOF
+            terraform {
+              required_providers {
+                #{name} = {
+                source = "#{source}"
+                version = "#{version}"
+                }
+              }
+            }
+        EOF
+      else
+        text +=<<~EOF
+            terraform {
+              required_providers {
+                #{name} = {
+                source = "#{source}"
+                }
+              }
+            }
+        EOF
+      end
+    end
+
+    text +=<<~EOF
+        provider "#{name}" {
+        #{variable_block(variables)}
+        }
+    EOF
+
+    element_file = ['provider_config', name.to_s].join('.')
+
+    custom(element_file, text)
+
+    provider
+  end
+
+  # Populate a directory with the terraform templates corresponding to the
+  # custom defined elements
+  #
+  # @param dir [String] directory from which to manage the deployment
+  def custom_files(dir)
+    @custom_files.each do |element_file, text|
+      Open.write(dir[element_file + '.tf'], text)
+    end
+  end
+
+  # Populate a directory with the terraform templates corresponding to the
+  # defined element outputs variables
+  #
+  # @param dir [String] directory from which to manage the deployment
+  def outputs(dir)
+    @elements.each do |info|
+      _provider, module_name, template, variables = info
+      outputs = variables[:outputs]
+      module_outputs = TerraformDSL.module_outputs(template)
+      outputs = module_outputs.keys if outputs.to_s == 'all'
+      next unless outputs && outputs.any?
+
+      name = variables[:name]
+
+      outputs = outputs.collect do |o|
+        (o.is_a?(String) || o.is_a?(Symbol)) && o.to_s == 'all' ? module_outputs.keys : o
+      end.flatten.uniq if outputs.is_a?(Array)
+
+      text = ''
+      outputs.each do |output, output_rename = nil|
+        output, output_rename = output.collect.first if output.is_a?(Hash)
+
+        output_rename = output if output_rename.nil?
+
+        description = module_outputs[output.to_s][:description]
+        description ||= "Value of #{output} from module #{name} (type #{module_name})"
+
+        text +=<<~EOF
+            output "#{name}_#{output_rename}"{
+              description = "#{description}"
+              value = module.#{name}.#{output}
+            }
+        EOF
+      end
+
+      element_file = [module_name, name.to_s] * "."
+
+      Open.write(dir[element_file + '.outputs.tf'], text)
+    end
+  end
+
+  # Populate a directory all the necessary templates: modules, outputs,
+  # and custom
+  #
+  # @param dir [String] directory from which to manage the deployment. If none
+  #   provided a unique one will be generated based on a md5 digest of
+  #   the elements defined
+  def config(dir = nil)
+    dir = WORK_DIR[TerraformDSL.obj2digest(@elements)] if dir.nil?
+    Open.mkdir dir
+    main(dir)
+    outputs(dir)
+    custom_files(dir)
+    @processed_elements ||= []
+    @processed_elements.concat(@elements)
+    @processed_custom_files ||= []
+    @processed_custom_files.concat(@custom_files)
+    @elements = []
+    @custom_files = []
+    dir
+  end
+
+end

data/lib/scout-camp.rb

@@ -0,0 +1,6 @@
+require 'scout'
+require 'scout/path'
+require 'scout/resource'
+Path.add_path :scout_camp, File.join(Path.caller_lib_dir(__FILE__), "{TOPLEVEL}/{SUBPATH}")
+require 'scout/terraform_dsl'
+require 'scout/offsite'

data/scout_commands/offsite

@@ -0,0 +1,30 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+require 'scout/offsite'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Description of the tool
+
+$ #{$0} [<options>] <where> <command> ... -- [<protected options>]
+
+Run a command offsite. Use -- to make sure options reach the offsite
+
+-h--help Print this help
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+where = ARGV.shift
+command = ARGV.shift
+
+puts SSHLine.command(where, command, ARGV)

data/scout_commands/terraform/add

@@ -0,0 +1,78 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Add a new deployment
+
+$ #{$0} [<options>] <name>
+
+-h--help Print this help
+-w--workflows* Workflows to provision
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+name = ARGV.shift
+raise MissingParameterException, :name if name.nil?
+
+deployments = Scout.var.deployments.glob_all("*").collect(&:basename)
+raise "Deployment clash" if deployments.include? name
+dir =  Scout.var.deployments[name]
+
+IndiferentHash.add_defaults options, workflows: '', dependencies: 'scout-gear'
+
+workflows = options[:workflows].split(",")
+dependencies = options[:dependencies].split(",")
+
+code =<<-EOF
+def lambda_handler(event:, context:)
+  require 'scout/workflow'
+
+  wf = Workflow.require_workflow event["workflow"]
+  job = wf.job(event["task_name"], event["jobname"], event["inputs"])
+  id = job.short_path
+  id
+end
+EOF
+
+def lambda_package(dependencies, workflows, code)
+  TmpFile.with_path do |dir|
+    dir["lambda_function.rb"].write code
+    dir["Gemfile"].write "source 'https://rubygems.org'\n" + dependencies.collect{|d| "gem '#{d}'"}*"\n"
+    workflows.each do |workflow|
+      Open.mkdir dir[".scout"].workflows
+      Open.cp Scout.workflows[workflow], dir[".scout"].workflows[workflow]
+    end
+    Misc.in_dir dir do
+      `bundle config set path 'vendor/bundle'`
+      `bundle install`
+      `rm -Rf vendor/bundle/ruby/*/cache`
+      `rm -Rf vendor/bundle/ruby/*/gems/RubyInline*`
+      TmpFile.with_file nil, extension: :zip do |zip|
+        `zip -r "#{zip}" vendor/bundle lambda_function.rb .scout`
+        yield zip
+      end
+    end
+  end
+end
+
+require 'aws-sdk'
+lambda_package dependencies, workflows, code do |file|
+  terraform = TerraformDSL.new
+  filename = 'lambda_package.zip'
+  Open.cp file, dir[filename]
+  terraform.add :aws, :lambda, function_name: "ScoutJob", filename: filename, environment_variables: {HOME: '.'}
+  terraform.config dir
+end
+
+

data/scout_commands/terraform/apply

@@ -0,0 +1,31 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Work with deployment
+
+$ #{$0} [<options>] <name>
+
+-h--help Print this help
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+name = ARGV.shift
+raise MissingParameterException, :name if name.nil?
+
+dir = Scout.var.deployments.glob_all(name).first
+
+deployment = TerraformDSL::Deployment.new dir
+deployment.apply
+

data/scout_commands/terraform/destroy

@@ -0,0 +1,31 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Work with deployment
+
+$ #{$0} [<options>] <name>
+
+-h--help Print this help
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+name = ARGV.shift
+raise MissingParameterException, :name if name.nil?
+
+dir = Scout.var.deployments.glob_all(name).first
+
+deployment = TerraformDSL::Deployment.new dir
+deployment.destroy
+

data/scout_commands/terraform/list

@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+List all deployments
+
+$ #{$0} [<options>] <filename>
+
+-h--help Print this help
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+deployments = Scout.var.deployments.glob_all("*")
+
+deployments.each do |dir|
+  name = File.basename dir
+  deployment = TerraformDSL::Deployment.new dir
+  active = deployment.provisioned_elements.any?
+  
+  if active
+    puts Log.color(:title, name ) + "\t" + dir + "\t" + "Active" 
+  else
+    puts Log.color(:title, name) + "\t" + dir
+  end
+end

data/scout_commands/terraform/remove

@@ -0,0 +1,39 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Work with deployment
+
+$ #{$0} [<options>] <name>
+
+-h--help Print this help
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+name = ARGV.shift
+raise MissingParameterException, :name if name.nil?
+
+dir = Scout.var.deployments.glob_all(name).first
+
+begin
+  deployment = TerraformDSL::Deployment.new dir
+  deployment.destroy
+rescue
+end
+
+if deployment.provisioned_elements.empty?
+  Open.rm_rf dir
+else
+  "Not removed"
+end

data/scout_commands/terraform/status

@@ -0,0 +1,33 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Work with deployment
+
+$ #{$0} [<options>] <name>
+
+-h--help Print this help
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+name = ARGV.shift
+raise MissingParameterException, :name if name.nil?
+
+dir = Scout.var.deployments.glob_all(name).first
+
+deployment = TerraformDSL::Deployment.new dir
+deployment.provisioned_elements.each do |element|
+  ppp deployment.element_state(element)
+end
+

data/share/terraform/aws/bucket/main.tf

@@ -0,0 +1,8 @@
+provider "aws" {
+  region = "eu-west-2"  # Change to your preferred AWS region
+}
+
+resource "aws_s3_bucket" "this" {
+  bucket = var.name  # Change to a globally unique name
+}
+

data/share/terraform/aws/bucket/output.tf

@@ -0,0 +1,3 @@
+output "bucket_name" {
+  value = aws_s3_bucket.my_bucket.id
+}

data/share/terraform/aws/bucket/variables.tf

@@ -0,0 +1,4 @@
+variable "name" {
+  description = "Bucket name"
+  type = string
+}

data/share/terraform/aws/cluster/main.tf

@@ -0,0 +1,66 @@
+locals {
+  cidr_block = "${var.cidr_block_base}/${var.cidr_block_mask}"
+  cidr_block_dest = "0.0.0.0/0"
+
+}
+
+resource "aws_vpc" "this" {
+    cidr_block = local.cidr_block
+
+    tags = {
+        Name = var.cidr_nametag
+    }
+}
+
+resource "aws_subnet" "this" {
+    vpc_id     = aws_vpc.this.id
+    cidr_block = local.cidr_block
+
+    map_public_ip_on_launch = var.map_public_ip_on_launch
+
+    tags = {
+        Name = var.subnet_nametag
+    }
+
+    availability_zone      = var.availability_zone
+}
+
+
+resource "aws_internet_gateway" "this" {
+    vpc_id = aws_vpc.this.id
+
+    tags = {
+        Name = var.gateway_nametag
+    }
+}
+
+resource "aws_route" "this" {
+    route_table_id         = aws_vpc.this.main_route_table_id
+    destination_cidr_block = local.cidr_block_dest
+    gateway_id             = aws_internet_gateway.this.id
+}
+
+resource "aws_security_group" "this" {
+  name        = "allow_all"
+  description = "Allow all traffic"
+  vpc_id     = aws_vpc.this.id
+
+  ingress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = [local.cidr_block_dest]
+  }
+
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = [local.cidr_block_dest]
+  }
+
+  tags = {
+    Name = var.security_group_nametag
+  }
+}
+

data/share/terraform/aws/cluster/output.tf

@@ -0,0 +1,9 @@
+output "aws_subnet_id" {
+  description = "Submet id"
+  value = aws_subnet.this.id
+}
+
+output "aws_security_group_id" {
+  description = "Security group id"
+  value = aws_security_group.this.id
+}