cloudshaper 0.0.4

data/lib/terraform_dsl.rb

@@ -0,0 +1,6 @@
+require 'terraform_dsl/stack'
+require 'terraform_dsl/stacks'
+require 'terraform_dsl/stack_module'
+require 'terraform_dsl/version'
+
+require 'tasks/tasks' if defined? Rake

data/lib/terraform_dsl/aws/remote_s3.rb

@@ -0,0 +1,20 @@
+module Terraform
+  module Aws
+    # Support AWS S3 remote state backend
+    module RemoteS3
+      private
+
+      def options_for_s3(command)
+        options = ''
+        options = "-backend=s3 #{config_opts_s3}" if command == :config
+        "terraform remote #{command} #{options}"
+      end
+
+      def config_opts_s3
+        config = "-backend-config='key=#{@stack.stack_id}' "
+        config += @stack.remote['s3'].map { |k, v| "-backend-config='#{k}=#{v}'" }.join(' ')
+        config
+      end
+    end
+  end
+end

data/lib/terraform_dsl/aws/tagging.rb

@@ -0,0 +1,23 @@
+module Terraform
+  # Aws provider-specific functionality, to be mixed in to stack elements
+  module Aws
+    def self.taggable?(resource_type)
+      supports_tagging = [:aws_autoscaling_group, :aws_customer_gateway,
+                          :aws_db_instance, :aws_elasticache_cluster,
+                          :aws_elb, :aws_instance, :aws_internet_gateway,
+                          :aws_network_acl, :aws_network_interface, :aws_route53_zone,
+                          :aws_route_table, :aws_s3_bucket, :aws_security_group,
+                          :aws_subnet, :aws_vpc, :aws_vpc_dhcp_options,
+                          :aws_vpc_peering_connection, :aws_vpn_connection,
+                          :aws_vpn_gateway]
+      supports_tagging.include?(resource_type.to_sym)
+    end
+
+    # Tag all resources (that support tagging) that we created with this stack id
+    def post_processing_aws
+      return unless Aws.taggable?(@resource_type)
+      @fields[:tags] ||= {}
+      @fields[:tags][:terraform_stack_id] = var(:terraform_stack_id)
+    end
+  end
+end

data/lib/terraform_dsl/command.rb

@@ -0,0 +1,58 @@
+module Terraform
+  # Wraps terraform command execution
+  class Command
+    attr_accessor :command
+
+    def initialize(stack, command)
+      @stack = stack
+      @command = options_for(command)
+      prepare
+    end
+
+    def env
+      vars = {}
+      @stack.variables.each { |k, v| vars["TF_VAR_#{k}"] = v[:default] }
+      @stack.module.secrets.each do |_provider, secrets|
+        secrets.each do |k, v|
+          vars[k.to_s] = v
+        end
+      end
+      vars
+    end
+
+    def execute
+      Process.waitpid(spawn(env, @command, chdir: @stack.stack_dir))
+      fail 'Command failed' unless $CHILD_STATUS.to_i == 0
+    end
+
+    protected
+
+    def prepare
+      FileUtils.mkdir_p(@stack.stack_dir)
+      File.open(File.join(@stack.stack_dir, 'terraform.tf.json'), 'w') { |f| f.write(generate) }
+    end
+
+    def options_for(cmd)
+      options = begin
+        case cmd
+        when :apply
+          '-input=false'
+        when :destroy
+          '-input=false -force'
+        when :plan
+          '-input=false -module-depth=-1'
+        when :graph
+          '-draw-cycles'
+        else
+          ''
+        end
+      end
+
+      "terraform #{cmd} #{options}"
+    end
+
+    def generate
+      @stack.module.generate
+    end
+  end
+end

data/lib/terraform_dsl/module.rb

@@ -0,0 +1,38 @@
+require 'terraform_dsl/stack_element'
+require 'terraform_dsl/stack_modules'
+require 'terraform_dsl/stack_module'
+require 'terraform_dsl/stacks'
+
+module Terraform
+  # Supports terraform 'modules'. In our case, we call them submodules because
+  # Module is a ruby keyword. We also support directly referencing other ruby-defined modules.
+  class Module < StackElement
+    def initialize(parent_module, module_name, &block)
+      super(parent_module, &block)
+
+      if @fields[:source].match(/^module_/)
+        build_submodule(parent_module, module_name)
+      end
+    end
+
+    private
+
+    def build_submodule(parent_module, module_name)
+      generated = generate_child_module(parent_module)
+      module_path = File.join(Stacks.dir, parent_module.id, module_name.to_s)
+      FileUtils.mkdir_p(module_path)
+      File.open(File.join(module_path, 'stack_module.tf.json'), 'w') { |f| f.write(generated) }
+      @fields[:source] = File.expand_path(module_path)
+    end
+
+    def generate_child_module(parent_module)
+      variables = @fields.clone
+      variables.delete(:source)
+      variables[:terraform_stack_id] = parent_module.id
+      child_name = @fields[:source].gsub(/^module_/, '')
+      child_module = StackModules.get(child_name)
+      child_module.build(variables)
+      child_module.generate
+    end
+  end
+end

data/lib/terraform_dsl/output.rb

@@ -0,0 +1,6 @@
+require 'terraform_dsl/stack_element'
+
+module Terraform
+  class Output < StackElement
+  end
+end

data/lib/terraform_dsl/provider.rb

@@ -0,0 +1,13 @@
+require 'terraform_dsl/secrets'
+require 'terraform_dsl/stack_element'
+
+module Terraform
+  # Implements DSL for a terraform provider, and a means of loading secrets.
+  class Provider < StackElement
+    def load_secrets(name)
+      @secrets ||= {}
+      @secrets[name.to_sym] = SECRETS[name.to_sym]
+      @secrets
+    end
+  end
+end

data/lib/terraform_dsl/remote.rb

@@ -0,0 +1,31 @@
+require 'terraform_dsl/aws/remote_s3'
+
+module Terraform
+  # Wrap 'remote' commands, such as config, pull, and push
+  # This allows us to store state remotely using different providers
+  class Remote < Command
+    class RemoteNotSupported < Exception; end
+    include Aws::RemoteS3
+    def initialize(stack, command)
+      super
+      unless @stack.remote.first
+        puts "\tWARNING: #{@stack.name} is not configured with a remote backend"
+        return
+      end
+
+      backend = @stack.remote.keys.first
+      sym = "options_for_#{backend}"
+
+      if self.respond_to?(sym, include_private: true)
+        @command = send(sym, command)
+      else
+        fail RemoteNotSupported, "Remote backend #{backend} is not supported yet"
+      end
+    end
+
+    def execute
+      return unless @stack.remote.first
+      super
+    end
+  end
+end

data/lib/terraform_dsl/resource.rb

@@ -0,0 +1,44 @@
+require 'terraform_dsl/stack_element'
+
+module Terraform
+  # Defines a terraform resource
+  class Resource < StackElement
+    attr_reader :resource_name
+
+    def initialize(parent_module, resource_name, resource_type, &block)
+      @resource_name = resource_name
+      @resource_type = resource_type
+      super(parent_module, &block)
+
+      # Allow provider specific post processing
+      sym = "post_processing_#{resource_type.split('_').first}"
+      send(sym) if self.respond_to?(sym, include_private: true)
+    end
+
+    # Allow provisioner blocks to be nested within resources
+    def provisioner(provisioner_type, &block)
+      provisioner_type = provisioner_type.to_sym
+
+      @fields[:provisioner] = @fields[:provisioner] || []
+
+      provisioner_set = Provisioner.new(@module, &block)
+      @fields[:provisioner] << { cleanup_provisioner_type(provisioner_type) => provisioner_set.fields }
+    end
+
+    private
+
+    def cleanup_provisioner_type(provisioner_type)
+      case provisioner_type.to_sym
+      when :remote_exec
+        'remote-exec'
+      when :local_exec
+        'local-exec'
+      else
+        provisioner_type
+      end
+    end
+  end
+  # Defines a terraform resource provisioner
+  class Provisioner < StackElement
+  end
+end

data/lib/terraform_dsl/secrets.rb

@@ -0,0 +1,29 @@
+require 'json'
+require 'open3'
+
+# Load and provide access to secrets required by terraform providers
+class SecretHash < Hash
+  class SecretNotFound < Exception; end
+
+  def initialize
+    super { |_secrets, key| fail SecretNotFound, "Secret `#{key}` not found" }
+  end
+end
+
+if ENV['TERRAFORM_ENV'] == 'test'
+  SECRETS ||= {
+    aws: {
+      access_key_id: 'foo',
+      secret_access_key: 'bar'
+    }
+  }
+else
+  SECRETS ||= begin
+    secrets_file = File.expand_path(ENV['CONFIG_PATH'] || 'config/secrets.json')
+    if File.exist?(secrets_file)
+      JSON.parse(File.read(secrets_file), symbolize_names: true, object_class: SecretHash)
+    else
+      {}
+    end
+  end
+end

data/lib/terraform_dsl/stack.rb

@@ -0,0 +1,79 @@
+require 'terraform_dsl/stacks'
+require 'terraform_dsl/stack_modules'
+require 'terraform_dsl/command'
+require 'terraform_dsl/remote'
+
+module Terraform
+  # Wrapper to instantiate a stack from a yaml definition
+  class Stack
+    class MalformedConfig < Exception; end
+    class << self
+      def load(config)
+        fail MalformedConfig, "Configuration malformed at #{config}" unless config.is_a?(Hash)
+        fail MalformedConfig, "A name must be specified for the stack #{config}" unless config.key?('name')
+        fail MalformedConfig, 'You must specify a uuid. Get one from rake uuid and add it to the config' unless config.key?('uuid')
+        new(config)
+      end
+    end
+
+    attr_reader :name, :description, :module,
+                :stack_dir, :stack_id, :remote
+
+    def initialize(config)
+      @name = config['name']
+      @uuid = config['uuid']
+      @description = config['description'] || ''
+      @variables = config['variables'] || {}
+      @remote = config['remote'] || {}
+      @stack_id = "terraform_#{@name}_#{@uuid}"
+      @module = StackModules.get(config['root'])
+      @variables['terraform_stack_id'] = @stack_id
+      @stack_dir = File.join(Stacks.dir, @stack_id)
+      @module.build(@variables.map { |k, v| [k.to_sym, v] }.to_h)
+    end
+
+    def apply
+      Command.new(self, :apply).execute
+    end
+
+    def destroy
+      Command.new(self, :destroy).execute
+    end
+
+    def plan
+      Command.new(self, :plan).execute
+    end
+
+    def get
+      Command.new(self, :get).execute
+    end
+
+    def show
+      Command.new(self, :show).execute
+    end
+
+    def pull
+      Remote.new(self, :pull).execute
+    end
+
+    def push
+      Remote.new(self, :pull).execute
+    end
+
+    def remote_config
+      Remote.new(self, :config).execute
+    end
+
+    def variables
+      @module.variables
+    end
+
+    def to_s
+      <<-eos
+Name: #{@name}
+Description: #{@description}
+Stack Directory: #{@stack_dir}
+      eos
+    end
+  end
+end

data/lib/terraform_dsl/stack_element.rb

@@ -0,0 +1,62 @@
+require 'terraform_dsl/aws/tagging'
+
+module Terraform
+  # Defines a single terraform stack element, subclass for any element defined in terraform DSL
+  class StackElement
+    include Aws
+    attr_reader :fields
+
+    def initialize(stack_module, &block)
+      @module = stack_module
+      @fields = {}
+      instance_eval(&block)
+    end
+
+    private
+
+    # Allows resource attributes to be specified with a nice syntax
+    # If the method is implemented, it will be treated as a nested resource
+    def method_missing(method_name, *args, &block)
+      symbol = method_name.to_sym
+      if args.length == 1
+        if args[0].nil?
+          fail "Passed nil to '#{method_name}'. Generally disallowed, subclass StackElement if you need this."
+        end
+        add_field(symbol, args[0])
+      else
+        add_field(symbol, Terraform::StackElement.new(@module, &block).fields)
+      end
+    end
+
+    # Get the runtime value of a variable
+    def get(variable_name)
+      @module.get(variable_name)
+    end
+
+    # Reference a variable
+    def var(variable_name)
+      "${var.#{variable_name}}"
+    end
+
+    # Syntax to handle interpolation of resource variables
+    def value_of(resource_type, resource_name, value_type)
+      "${#{resource_type}.#{resource_name}.#{value_type}}"
+    end
+
+    # Shorthand to interpolate the ID of another resource
+    def id_of(resource_type, resource_name)
+      value_of(resource_type, resource_name, :id)
+    end
+
+    def add_field(symbol, value)
+      existing = @fields[symbol]
+      if existing
+        # If it's already an array, just push to it
+        @fields[symbol] = [existing] unless existing.is_a?(Array)
+        @fields[symbol] << value
+      else
+        @fields[symbol] = value
+      end
+    end
+  end
+end

data/lib/terraform_dsl/stack_module.rb

@@ -0,0 +1,121 @@
+require 'json'
+require 'fileutils'
+
+require 'terraform_dsl/stack_modules'
+require 'terraform_dsl/resource'
+require 'terraform_dsl/provider'
+require 'terraform_dsl/variable'
+require 'terraform_dsl/module'
+require 'terraform_dsl/output'
+
+module Terraform
+  # Stack Modules contain stack elements. A stack is made up of a root module, which may have submodules
+  class StackModule
+    class << self
+      def define(name, &block)
+        template = new(name, &block)
+        StackModules.register(name, template)
+      end
+
+      def flatten_variable_arrays(variables)
+        vars = variables.map do |k, v|
+          if v.is_a?(Hash) && v.key?(:default) && v[:default].is_a?(Array)
+            v[:default] = v[:default].join(',')
+          elsif v.is_a?(Array)
+            v = v.join(',')
+          end
+          [k, v]
+        end
+        Hash[vars]
+      end
+    end
+
+    attr_accessor :secrets
+
+    def initialize(_name, &block)
+      @stack_elements = { resource: {}, provider: {}, variable: {}, output: {}, module: {} }
+      @secrets = {}
+      @block = block
+      variable(:terraform_stack_id) {}
+    end
+
+    def clone
+      b = @block
+      StackModule.new(@name, &b)
+    end
+
+    def build(**kwargs)
+      vars = Hash[kwargs.map { |k, v| [k, { default: v }] }]
+      @stack_elements[:variable].merge!(vars)
+      b = @block
+      instance_eval(&b)
+    end
+
+    def generate
+      JSON.pretty_generate(elements)
+    end
+
+    def variables
+      elements[:variable]
+    end
+
+    def outputs
+      @stack_elements[:output]
+    end
+
+    def get(variable)
+      @stack_elements[:variable].fetch(variable)[:default]
+    end
+
+    def elements
+      elements = @stack_elements.clone
+      variables = StackModule.flatten_variable_arrays(@stack_elements[:variable])
+      @stack_elements[:module].each do |mod, data|
+        elements[:module][mod] = StackModule.flatten_variable_arrays(data)
+      end
+      elements[:variable] = variables
+      elements
+    end
+
+    def id
+      get(:terraform_stack_id)
+    end
+
+    private
+
+    def register_resource(resource_type, name, &block)
+      @stack_elements[:resource] ||= {}
+      @stack_elements[:resource][resource_type.to_sym] ||= {}
+      @stack_elements[:resource][resource_type.to_sym][name.to_sym] = Terraform::Resource.new(self, name, resource_type, &block).fields
+    end
+
+    def register_variable(name, &block)
+      new_variable = Terraform::Variable.new(self, &block).fields
+      unless @stack_elements[:variable].key?(name.to_sym)
+        @stack_elements[:variable][name.to_sym] = { default: new_variable[:default] || '' }
+      end
+    end
+
+    def register_output(name, &block)
+      new_output = Terraform::Output.new(self, &block).fields
+      @stack_elements[:output][name.to_sym] = new_output
+    end
+
+    def register_module(name, &block)
+      new_module = Terraform::Module.new(self, name, &block).fields
+      @stack_elements[:module][name.to_sym] = new_module
+    end
+
+    def register_provider(name, &block)
+      provider = Terraform::Provider.new(self, &block)
+      @secrets.merge!(provider.load_secrets(name))
+      @stack_elements[:provider][name.to_sym] = provider.fields
+    end
+
+    alias_method :resource, :register_resource
+    alias_method :variable, :register_variable
+    alias_method :provider, :register_provider
+    alias_method :output,   :register_output
+    alias_method :submodule,    :register_module
+  end
+end