TerraformDevKit 0.1.14 → 0.2.0

data/lib/TerraformDevKit/command.rb

@@ -2,48 +2,25 @@ require 'open3'
 
 module TerraformDevKit
   class Command
-    def self.run(cmd, directory: Dir.pwd, print_output: true, close_stdin: true)
-      output = []
-
-      Open3.popen2e(cmd, chdir: directory) do |stdin, stdout_and_stderr, thread|
-        stdout_thread = Thread.new do
-          process_output(stdout_and_stderr, print_output, output)
-        end
-
-        if close_stdin
-          stdin.close
-        else
-          input_thread = Thread.new do
-            loop { stdin.puts $stdin.gets }
-          end
-        end
+    def self.run(cmd, directory: Dir.pwd, print_output: true)
+      Open3.popen2e(cmd, chdir: directory) do |_, stdout_and_stderr, thread|
+        output = process_output(stdout_and_stderr, print_output)
 
         thread.join
-        stdout_thread.join
-        input_thread.terminate unless close_stdin
         raise "Error running command #{cmd}" unless thread.value.success?
+        return output
       end
-
-      output
     end
 
     private_class_method
-    def self.process_output(stdout_and_stderr, print_output, output)
-      line = ''
-      stdout_and_stderr.each_char do |char|
-        $stdout.print(char) if print_output
-        case char
-        when "\r"
-          next
-        when "\n"
-          output << line
-          line = ''
-        else
-          line << char
-        end
-      end
+    def self.process_output(stream, print_output)
+      lines = []
 
-      output << line unless line.empty?
+      until (line = stream.gets).nil?
+        print line if print_output
+        lines << line.strip
+      end
+      lines
     end
   end
 end

data/lib/TerraformDevKit/dynamodb.rb

@@ -3,9 +3,10 @@ require 'aws-sdk'
 Aws.use_bundled_cert!
 
 module TerraformDevKit
+  # Wrapper class around aws dynamodb
   class DynamoDB
     def initialize(credentials, region)
-      @db_client = Aws::DynamoDB::Resource.new(
+      @db_client = Aws::DynamoDB::Client.new(
         credentials: credentials,
         region: region
       )
@@ -20,5 +21,30 @@ module TerraformDevKit
       table = @db_client.table(table_name)
       table.put_item(item: item)
     end
+
+    def create_table(table_name, attributes, keys, read_capacity, write_capacity)
+      @db_client.create_table(
+        attribute_definitions: attributes,
+        key_schema: keys,
+        provisioned_throughput: {
+          read_capacity_units: read_capacity,
+          write_capacity_units: write_capacity
+        },
+        table_name: table_name
+      )
+    end
+
+    def get_table_status(table_name)
+      resp = @db_client.describe_table({
+        table_name: table_name,
+      })
+      resp.table.table_status
+    end
+
+    def delete_table(table_name)
+      @db_client.delete_table({
+        table_name: table_name,
+      })
+      end
   end
 end

data/lib/TerraformDevKit/s3.rb

@@ -0,0 +1,43 @@
+require 'aws-sdk'
+
+Aws.use_bundled_cert!
+
+module TerraformDevKit
+  # Wrapper class around aws s3
+  class S3
+    def initialize(credentials, region)
+      @s3_client = Aws::S3::Client.new(
+        credentials: credentials,
+        region: region
+      )
+    end
+
+    def create_bucket(bucket_name)
+      @s3_client.create_bucket(
+        bucket: bucket_name
+      )
+    end
+
+    def delete_bucket(bucket_name)
+      empty_bucket(bucket_name)
+
+      @s3_client.delete_bucket(
+        bucket: bucket_name
+      )
+    end
+
+    def empty_bucket(bucket_name)
+      keys_to_delete = @s3_client
+                       .list_objects_v2(bucket: bucket_name)
+                       .contents
+                       .map { |x| { key: x.key } }
+
+      @s3_client.delete_objects(
+        bucket: bucket_name,
+        delete: {
+          objects: keys_to_delete
+        }
+      )
+    end
+  end
+end

data/lib/TerraformDevKit/terraform_config_manager.rb

@@ -1,6 +1,6 @@
-require 'TerraformDevKit/terraform_template_config_file'
-
 require 'fileutils'
+require 'TerraformDevKit/terraform_project_config'
+require 'TerraformDevKit/terraform_template_config_file'
 
 module TerraformDevKit
   class TerraformConfigManager
@@ -10,10 +10,10 @@ module TerraformDevKit
       @extra_vars_proc = p
     end
 
-    def self.setup(env)
+    def self.setup(env, project)
       fix_configuration(env)
       create_environment_directory(env)
-      render_template_config_files(env)
+      render_template_config_files(env, project)
     end
 
     def self.update_modules?
@@ -42,12 +42,13 @@ module TerraformDevKit
     end
 
     private_class_method
-    def self.render_template_config_files(env)
+    def self.render_template_config_files(env, project)
       aws_config = Configuration.get('aws')
       file_list = Dir['*.tf.mustache'] + Dir['*.tfvars.mustache']
       file_list.each do |fname|
         template_file = TerraformTemplateConfigFile.new(
           File.read(fname),
+          project,
           env,
           aws_config,
           extra_vars: @extra_vars_proc.call(env)

data/lib/TerraformDevKit/terraform_project_config.rb

@@ -0,0 +1,10 @@
+module TerraformDevKit
+  class TerraformProjectConfig
+    attr_reader :name, :acronym
+    
+    def initialize(project_name)
+      @name = project_name.gsub(' ', '-').downcase
+      @acronym = project_name.scan(/\b[a-z]/i).join.upcase
+    end
+  end
+end

data/lib/TerraformDevKit/terraform_remote_state.rb

@@ -0,0 +1,65 @@
+require 'aws-sdk'
+
+module TerraformDevKit
+  # Represents a terraform lock table.
+  class TerraformRemoteState
+    ATTRIBUTES = [
+        {
+          attribute_name: 'LockID',
+          attribute_type: 'S'
+        }
+      ]
+    KEYS = [
+        {
+          attribute_name: 'LockID',
+          key_type: 'HASH'
+        }
+      ]
+
+    def initialize(dynamodb, s3)
+      @dynamodb = dynamodb
+      @s3 = s3
+    end
+
+    def init(environment, project)
+      table_name = table_name(environment, project)
+      return if lock_table_exists_and_is_active(table_name)
+
+      @dynamodb.create_table(table_name, ATTRIBUTES, KEYS, 1, 1)
+
+      begin
+        @s3.create_bucket(state_bucket_name(environment, project))
+      rescue Aws::S3::Errors::BucketAlreadyOwnedByYou
+        return
+      end
+
+      sleep(0.2) until lock_table_exists_and_is_active(table_name)
+    end
+
+    def destroy(environment, project)
+      table_name = table_name(environment, project)
+
+      @dynamodb.delete_table(table_name)
+      @s3.delete_bucket(state_bucket_name(environment, project))
+    end
+
+    private_class_method
+    def lock_table_exists_and_is_active(table_name)
+      begin
+        return @dynamodb.get_table_status(table_name) == 'ACTIVE'
+      rescue Aws::DynamoDB::Errors::ResourceNotFoundException
+        return false
+      end
+    end
+
+    private_class_method
+    def table_name(environment, project)
+      "#{project.acronym}-#{environment.name}-lock-table"
+    end
+
+    private_class_method
+    def state_bucket_name(environment, project)
+      "#{project.name}-#{environment.name}-state"
+    end
+  end
+end

data/lib/TerraformDevKit/terraform_template_config_file.rb

@@ -2,8 +2,9 @@ require 'mustache'
 
 module TerraformDevKit
   class TerraformTemplateConfigFile
-    def initialize(content, env, aws_config, extra_vars: {})
+    def initialize(content, project, env, aws_config, extra_vars: {})
       @content = content
+      @project = project
       @env = env
       @aws_config = aws_config
       @extra_vars = extra_vars
@@ -14,7 +15,9 @@ module TerraformDevKit
         Profile: @aws_config.fetch('profile', ''),
         Region:  @aws_config.fetch('region'),
         Environment: @env.name,
-        LocalBackend: @env.local_backend?
+        LocalBackend: @env.local_backend?,
+        ProjectName: @project.name,
+        ProjectAcronym: @project.acronym
       }
       args.merge!(@extra_vars)
       Mustache.render(

data/lib/TerraformDevKit/version.rb

@@ -1,3 +1,3 @@
-module TerraformDevKit
-  VERSION = '0.1.14'.freeze
-end
+module TerraformDevKit
+  VERSION = '0.2.0'.freeze
+end

data/lib/TerraformDevKit/zip_file_generator.rb

@@ -1,47 +1,47 @@
-require 'zip'
-
-module TerraformDevKit
-  class ZipFileGenerator
-    def initialize(input_dir, output_file)
-      @input_dir = input_dir
-      @output_file = output_file
-    end
-
-    def write
-      entries = Dir.entries(@input_dir)
-      entries.delete('.')
-      entries.delete('..')
-      Zip::File.open(@output_file, Zip::File::CREATE) do |zipfile|
-        write_entries(entries, '', zipfile)
-      end
-    end
-
-    private
-
-    def write_entries(entries, path, zipfile)
-      entries.each do |e|
-        zip_file_path = path == '' ? e : File.join(path, e)
-        disk_file_path = File.join(@input_dir, zip_file_path)
-        if File.directory?(disk_file_path)
-          write_directory(disk_file_path, zip_file_path, zipfile)
-        else
-          write_file(disk_file_path, zip_file_path, zipfile)
-        end
-      end
-    end
-
-    def write_directory(disk_file_path, zip_file_path, zipfile)
-      zipfile.mkdir(zip_file_path)
-      subdir = Dir.entries(disk_file_path)
-      subdir.delete('.')
-      subdir.delete('..')
-      write_entries(subdir, zip_file_path, zipfile)
-    end
-
-    def write_file(disk_file_path, zip_file_path, zipfile)
-      zipfile.get_output_stream(zip_file_path) do |f|
-        f.puts(File.open(disk_file_path, 'rb').read)
-      end
-    end
-  end
-end
+require 'zip'
+
+module TerraformDevKit
+  class ZipFileGenerator
+    def initialize(input_dir, output_file)
+      @input_dir = input_dir
+      @output_file = output_file
+    end
+
+    def write
+      entries = Dir.entries(@input_dir)
+      entries.delete('.')
+      entries.delete('..')
+      Zip::File.open(@output_file, Zip::File::CREATE) do |zipfile|
+        write_entries(entries, '', zipfile)
+      end
+    end
+
+    private
+
+    def write_entries(entries, path, zipfile)
+      entries.each do |e|
+        zip_file_path = path == '' ? e : File.join(path, e)
+        disk_file_path = File.join(@input_dir, zip_file_path)
+        if File.directory?(disk_file_path)
+          write_directory(disk_file_path, zip_file_path, zipfile)
+        else
+          write_file(disk_file_path, zip_file_path, zipfile)
+        end
+      end
+    end
+
+    def write_directory(disk_file_path, zip_file_path, zipfile)
+      zipfile.mkdir(zip_file_path)
+      subdir = Dir.entries(disk_file_path)
+      subdir.delete('.')
+      subdir.delete('..')
+      write_entries(subdir, zip_file_path, zipfile)
+    end
+
+    def write_file(disk_file_path, zip_file_path, zipfile)
+      zipfile.get_output_stream(zip_file_path) do |f|
+        f.puts(File.open(disk_file_path, 'rb').read)
+      end
+    end
+  end
+end

data/lib/TerraformDevKit.rb

@@ -1,17 +1,19 @@
-require 'TerraformDevKit/aws'
-require 'TerraformDevKit/backup_state'
-require 'TerraformDevKit/command'
-require 'TerraformDevKit/config'
-require 'TerraformDevKit/dynamodb'
-require 'TerraformDevKit/environment'
-require 'TerraformDevKit/os'
-require 'TerraformDevKit/request'
-require 'TerraformDevKit/retry'
-require 'TerraformDevKit/terraform_config_manager'
-require 'TerraformDevKit/terraform_env_manager'
-require 'TerraformDevKit/terraform_installer'
-require 'TerraformDevKit/terraform_log_filter'
-require 'TerraformDevKit/terraform_template_config_file'
-require 'TerraformDevKit/terragrunt_installer'
-require 'TerraformDevKit/url'
-require 'TerraformDevKit/version'
+require 'TerraformDevKit/aws'
+require 'TerraformDevKit/backup_state'
+require 'TerraformDevKit/command'
+require 'TerraformDevKit/config'
+require 'TerraformDevKit/dynamodb'
+require 'TerraformDevKit/environment'
+require 'TerraformDevKit/os'
+require 'TerraformDevKit/request'
+require 'TerraformDevKit/retry'
+require 'TerraformDevKit/s3'
+require 'TerraformDevKit/terraform_config_manager'
+require 'TerraformDevKit/terraform_env_manager'
+require 'TerraformDevKit/terraform_installer'
+require 'TerraformDevKit/terraform_remote_state'
+require 'TerraformDevKit/terraform_log_filter'
+require 'TerraformDevKit/terraform_project_config'
+require 'TerraformDevKit/terraform_template_config_file'
+require 'TerraformDevKit/url'
+require 'TerraformDevKit/version'

data/tasks/devkit.rake

@@ -1,4 +1,5 @@
 require 'fileutils'
+require 'rainbow'
 require 'TerraformDevKit'
 
 TDK = TerraformDevKit
@@ -6,12 +7,14 @@ TDK = TerraformDevKit
 raise 'ROOT_PATH is not defined' if defined?(ROOT_PATH).nil?
 BIN_PATH = File.join(ROOT_PATH, 'bin')
 
-# Ensure terraform and terragrunt are in the PATH
+# Ensure terraform is in the PATH
 ENV['PATH'] = TDK::OS.join_env_path(
   TDK::OS.convert_to_local_path(BIN_PATH),
   ENV['PATH']
 )
 
+PLAN_FILE = 'plan.tfplan'.freeze
+
 def destroy_if_fails(env)
   yield
 rescue StandardError => e
@@ -22,9 +25,20 @@ rescue StandardError => e
 end
 
 def invoke_if_defined(task_name, env)
-  if Rake::Task.task_defined?(task_name)
-    task(task_name).invoke(env)
-  end
+  task(task_name).invoke(env) if Rake::Task.task_defined?(task_name)
+end
+
+def remote_state
+  aws_config = TDK::AwsConfig.new(TDK::Configuration.get('aws'))
+  dynamo_db = TDK::DynamoDB.new(
+    aws_config.credentials,
+    aws_config.region
+  )
+  s3 = TDK::S3.new(
+    aws_config.credentials,
+    aws_config.region
+  )
+  TDK::TerraformRemoteState.new(dynamo_db, s3)
 end
 
 desc 'Prepares the environment to create the infrastructure'
@@ -40,30 +54,35 @@ task :prepare, [:env] do |_, args|
     TDK::Configuration.get('terraform-version'),
     directory: BIN_PATH
   )
-  TDK::TerragruntInstaller.install_local(
-    TDK::Configuration.get('terragrunt-version'),
-    directory: BIN_PATH
+
+  project_config = TDK::TerraformProjectConfig.new(
+    TDK::Configuration.get('project-name')
   )
+  TDK::TerraformConfigManager.setup(env, project_config)
 
-  TDK::TerraformConfigManager.setup(env)
+  unless env.local_backend?
+    puts '== Initializing remote state'
+    remote_state.init(env, project_config)
+  end
 
   invoke_if_defined('custom_prepare', args.env)
 
-  TDK::Command.run(
-    'terragrunt init -upgrade=false',
-    directory: env.working_dir,
-    close_stdin: false
-  )
+  if File.exist?(File.join(env.working_dir, '.terraform'))
+    get_cmd  = 'terraform get'
+    get_cmd += ' -update=true' if TDK::TerraformConfigManager.update_modules?
+    TDK::Command.run(get_cmd, directory: env.working_dir)
+  else
+    init_cmd  = 'terraform init'
+    init_cmd += ' -upgrade=false' unless TDK::TerraformConfigManager.update_modules?
 
-  cmd  = 'terragrunt get'
-  cmd += ' -update=true' if TDK::TerraformConfigManager.update_modules?
-  TDK::Command.run(cmd, directory: env.working_dir)
+    TDK::Command.run(init_cmd, directory: env.working_dir)
+  end
 end
 
 desc 'Shows the plan to create the infrastructure'
 task :plan, [:env] => :prepare do |_, args|
   env = TDK::Environment.new(args.env)
-  TDK::Command.run('terragrunt plan', directory: env.working_dir)
+  TDK::Command.run("terraform plan -out=#{PLAN_FILE}", directory: env.working_dir)
 end
 
 desc 'Creates the infrastructure'
@@ -71,8 +90,21 @@ task :apply, [:env] => :prepare do |_, args|
   invoke_if_defined('pre_apply', args.env)
 
   env = TDK::Environment.new(args.env)
+
+  task('plan').invoke(env.name)
+
+  unless env.local_backend?
+    puts Rainbow("Are you sure you want to apply the above plan?\n" \
+                 "Only 'yes' will be accepted.").green
+    response = STDIN.gets.strip
+    unless response == 'yes'
+      raise "Apply cancelled because response was not 'yes'.\n" \
+            "Response was: #{response}"
+    end
+  end
+
   destroy_if_fails(env) do
-    TDK::Command.run('terragrunt apply', directory: env.working_dir)
+    TDK::Command.run("terraform apply \"#{PLAN_FILE}\"", directory: env.working_dir)
   end
 
   invoke_if_defined('post_apply', args.env)
@@ -83,7 +115,7 @@ task :test, [:env] do |_, args|
   env = TDK::Environment.new(args.env)
   env.local_backend? || (raise 'Testing is only allowed for local environments')
 
-  task('apply').invoke(env.name)
+  task('apply').invoke(env.name, true)
 
   destroy_if_fails(env) do
     invoke_if_defined('custom_test', args.env)
@@ -103,9 +135,32 @@ task :destroy, [:env] => :prepare do |_, args|
   invoke_if_defined('pre_destroy', args.env)
 
   env = TDK::Environment.new(args.env)
-  cmd = 'terragrunt destroy'
-  cmd += ' -force' if env.local_backend?
-  TDK::Command.run(cmd, directory: env.working_dir, close_stdin: false)
+  cmd = 'terraform destroy'
+
+  unless env.local_backend?
+    puts Rainbow("\n\n!!!! WARNING !!!!\n\n" \
+                 "You are about to destroy #{env.name} and its remote state.\n" \
+                 "Are you sure you want to proceed?\n" \
+                 "Only 'yes' will be accepted.").red.bright
+    response = STDIN.gets.strip
+
+    unless response == 'yes'
+      raise "Destroy cancelled because response was not 'yes'.\n" \
+           "Response was: #{response}"
+    end
+  end
+  
+  cmd += ' -force'
+
+  TDK::Command.run(cmd, directory: env.working_dir)
+  invoke_if_defined('pre_destroy', args.env)
+
+  unless env.local_backend?
+    project_config = TDK::TerraformProjectConfig.new(
+      TDK::Configuration.get('project-name')
+    )
+    remote_state.destroy(env, project_config)
+  end
 
   invoke_if_defined('post_destroy', args.env)
 end