tfctl 1.0.0

data/lib/tfctl/config.rb

@@ -0,0 +1,182 @@
+# frozen_string_literal: true
+
+require_relative '../hash.rb'
+require_relative 'error.rb'
+require 'yaml'
+require 'json'
+
+module Tfctl
+    class Config
+        include Enumerable
+        attr_reader :config
+
+        def initialize(config_name:, yaml_config:, aws_org_config:, use_cache: false)
+            cache_file  = "#{PROJECT_ROOT}/.tfctl/#{config_name}_cache.yaml"
+
+            # Get configuration.  Either load from cache or process fresh.
+            if use_cache
+                @config = read_cache(cache_file)
+            else
+                @config = load_config(config_name, yaml_config, aws_org_config)
+                write_cache(cache_file)
+            end
+        end
+
+        def [](key)
+            @config[key]
+        end
+
+        def fetch(key, default)
+            @config.fetch(key, default)
+        end
+
+        def each(&block)
+            @config.each(&block)
+        end
+
+        def key?(key)
+            @config.key?(key)
+        end
+
+        alias has_key? key?
+
+        def to_yaml
+            @config.to_yaml
+        end
+
+        def to_json(*_args)
+            @config.to_json
+        end
+
+        # Filters accounts by account property
+        def find_accounts(property_name, property_value)
+            output =[]
+            @config[:accounts].each do |account|
+                if account[property_name] == property_value
+                    output << account
+                end
+            end
+
+            if output.empty?
+                raise Tfctl::Error, "Account not found with #{property_name}: #{property_value}"
+            end
+
+            output
+        end
+
+        def find_accounts_regex(property_name, expr)
+            output =[]
+            @config[:accounts].each do |account|
+                begin
+                    if account[property_name] =~ /#{expr}/
+                        output << account
+                    end
+                rescue RegexpError => e
+                    raise Tfctl::Error, "Regexp: #{e}"
+                end
+            end
+
+            if output.empty?
+                raise Tfctl::Error, "Account not found with #{property_name} matching regex: #{expr}"
+            end
+
+            output
+        end
+
+
+        private
+
+        # Retrieves AWS Organizations data and merges it with data from yaml config.
+        def load_config(config_name, yaml_config, aws_org_config)
+
+            # AWS Organizations data
+            config = aws_org_config
+            # Merge organization sections from yaml file
+            config = merge_accounts_config(config, yaml_config)
+            # Import remaining parameters from yaml file
+            config = import_yaml_config(config, yaml_config)
+            # Set excluded property on any excluded accounts
+            config = mark_excluded_accounts(config)
+            # Remove any profiles that are unset
+            config = remove_unset_profiles(config)
+            # Set config name property (based on yaml config file name)
+            config[:config_name] = config_name
+            config
+        end
+
+        def write_cache(cache_file)
+            FileUtils.mkdir_p File.dirname(cache_file)
+            File.open(cache_file, 'w') { |f| f.write to_yaml }
+        end
+
+        def read_cache(cache_file)
+            unless File.exist?(cache_file)
+                raise Tfctl::Error, "Cached configuration not found in: #{cache_file}"
+            end
+
+            YAML.load_file(cache_file)
+        end
+
+        # Sets :excluded property on any excluded accounts
+        def mark_excluded_accounts(config)
+            return config unless config.key?(:exclude_accounts)
+
+            config[:accounts].each_with_index do |account, idx|
+                config[:accounts][idx][:excluded] = config[:exclude_accounts].include?(account[:name]) ? true : false
+            end
+
+            config
+        end
+
+        def remove_unset_profiles(config)
+            config[:accounts].each do |account|
+                profiles_to_unset = []
+                account[:profiles].each do |profile|
+                    if profile =~  /\.unset$/
+                        profiles_to_unset << profile
+                        profiles_to_unset << profile.chomp('.unset')
+                    end
+                end
+                account[:profiles] = account[:profiles] - profiles_to_unset
+            end
+            config
+        end
+
+        # Import yaml config other than organisation defaults sections which are merged elsewhere.
+        def import_yaml_config(config, yaml_config)
+            yaml_config.delete(:organization_root)
+            yaml_config.delete(:organization_units)
+            yaml_config.delete(:account_overrides)
+            config.merge(yaml_config)
+        end
+
+        # Merge AWS Organizations accounts config with defaults from yaml config
+        def merge_accounts_config(config, yaml_config)
+
+            config[:accounts].each_with_index do |account_config, idx|
+                account_name       = account_config[:name].to_sym
+                account_ou_parents = account_config[:ou_parents]
+
+                # merge any root settings
+                account_config = account_config.deep_merge(yaml_config[:organization_root])
+
+                # merge all OU levels settings
+                account_ou_parents.each_with_index do |_, i|
+                    account_ou = account_ou_parents[0..i].join('/').to_sym
+                    if yaml_config[:organization_units].key?(account_ou)
+                        account_config = account_config.deep_merge(yaml_config[:organization_units][account_ou])
+                    end
+                end
+
+                # merge any account overrides
+                if yaml_config[:account_overrides].key?(account_name)
+                    account_config = account_config.deep_merge(yaml_config[:account_overrides][account_name])
+                end
+
+                config[:accounts][idx] = account_config
+            end
+            config
+        end
+
+    end
+end

data/lib/tfctl/error.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Tfctl
+    class Error < StandardError
+    end
+
+    class ValidationError < StandardError
+        attr_reader :issues
+
+        def initialize(message, issues = [])
+            super(message)
+            @issues = issues
+        end
+    end
+end

data/lib/tfctl/executor.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+require 'open3'
+require 'fileutils'
+require 'shellwords'
+require_relative 'error.rb'
+
+module Tfctl
+    module Executor
+        module_function
+
+        # Execute terraform command
+        def run(account_name:, config_name:, log:, cmd: nil, argv: [], unbuffered: true)
+
+            # Use bin/terraform from a project dir if available
+            # Otherwise rely on PATH.
+            if cmd.nil?
+                cmd = File.exist?("#{PROJECT_ROOT}/bin/terraform") ? "#{PROJECT_ROOT}/bin/terraform" : 'terraform'
+            end
+
+            # Fail if there are no arguments for terraform and show terraform -help
+            if argv.empty?
+                help = `#{cmd} -help`.lines.to_a[1..-1].join
+                raise Tfctl::Error, "Missing terraform command.\n #{help}"
+            end
+
+            path       = "#{PROJECT_ROOT}/.tfctl/#{config_name}/#{account_name}"
+            cwd        = FileUtils.pwd
+            plan_file  = "#{path}/tfplan"
+            semaphore  = Mutex.new
+            output     = []
+
+            # Extract terraform sub command from argument list
+            args       = Array.new(argv)
+            subcmd     = args[0]
+            args.delete_at(0)
+
+            # Enable plan file for `plan` and `apply` sub commands
+            args += plan_file_args(plan_file, subcmd)
+
+            # Create the command
+            exec = [cmd] + [subcmd] + args
+
+            # Set environment variables for Terraform
+            env = {
+                'TF_INPUT'           => '0',
+                'CHECKPOINT_DISABLE' => '1',
+                'TF_IN_AUTOMATION'   => 'true',
+                # 'TF_LOG'             => 'TRACE'
+            }
+
+            log.debug "#{account_name}: Executing: #{exec.shelljoin}"
+
+            FileUtils.cd path
+            Open3.popen3(env, exec.shelljoin) do |stdin, stdout, stderr, wait_thr|
+                stdin.close_write
+
+                # capture stdout and stderr in separate threads to prevent deadlocks
+                Thread.new do
+                    stdout.each do |line|
+                        semaphore.synchronize do
+                            unbuffered ? log.info("#{account_name}: #{line.chomp}") : output << ['info', line]
+                        end
+                    end
+                end
+                Thread.new do
+                    stderr.each do |line|
+                        semaphore.synchronize do
+                            unbuffered ? log.error("#{account_name}: #{line.chomp}") : output << ['error', line]
+                        end
+                    end
+                end
+
+                status = wait_thr.value
+
+                # log the output
+                output.each do |line|
+                    log.send(line[0], "#{account_name}: #{line[1].chomp}")
+                end
+
+                FileUtils.cd cwd
+                FileUtils.rm_f plan_file if args[0] == 'apply' # tidy up the plan file
+
+                unless status.exitstatus.zero?
+                    raise Tfctl::Error, "#{cmd} failed with exit code: #{status.exitstatus}"
+                end
+            end
+        end
+
+        # Adds plan file to `plan` and `apply` sub commands
+        def plan_file_args(plan_file, subcmd)
+            return ["-out=#{plan_file}"] if subcmd == 'plan'
+
+            if subcmd == 'apply'
+                raise Tfctl::Error, "Plan file not found in #{plan_file}.  Run plan first." unless File.exist?(plan_file)
+
+                return [plan_file.to_s]
+            end
+
+            return []
+        end
+    end
+end

data/lib/tfctl/generator.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+
+# Generates top level Terraform configuration for an account.
+
+module Tfctl
+    module Generator
+        module_function
+
+        def write_json_block(path, block)
+            File.open(path, 'w') do |f|
+                f.write(JSON.pretty_generate(block) + "\n")
+            end
+        end
+
+        def make(account:, config:)
+            target_dir = "#{PROJECT_ROOT}/.tfctl/#{config[:config_name]}/#{account[:name]}"
+            tf_version = config.fetch(:tf_required_version, '>= 0.12.0')
+            aws_provider_version = config.fetch(:aws_provider_version, '>= 2.14')
+
+            FileUtils.mkdir_p target_dir
+
+            terraform_block = {
+                'terraform' => {
+                    'required_version' => tf_version,
+                    'backend'          => {
+                        's3' => {
+                            'bucket'         => config[:tf_state_bucket],
+                            'key'            => "#{account[:name]}/tfstate",
+                            'region'         => config[:tf_state_region],
+                            'role_arn'       => config[:tf_state_role_arn],
+                            'dynamodb_table' => config[:tf_state_dynamodb_table],
+                            'encrypt'        => 'true',
+                        },
+                    },
+                },
+            }
+            write_json_block("#{target_dir}/terraform.tf.json", terraform_block)
+
+            provider_block = {
+                'provider' => {
+                    'aws' => {
+                        'version'     => aws_provider_version,
+                        'region'      => account[:region],
+                        'assume_role' => {
+                            'role_arn' => "arn:aws:iam::#{account[:id]}:role/#{account[:tf_execution_role]}",
+                        },
+                    },
+                },
+            }
+            write_json_block("#{target_dir}/provider.tf.json", provider_block)
+
+            vars_block = {
+                'variable' => {
+                    'config' => {
+                        'type' => 'string',
+                    },
+                },
+            }
+            write_json_block("#{target_dir}/vars.tf.json", vars_block)
+
+            # config is passed to profiles as a json encoded string.  It can be
+            # decoded in profile using jsondecode() function.
+            config_block = { 'config' => config.to_json }
+            write_json_block("#{target_dir}/config.auto.tfvars.json", config_block)
+
+            FileUtils.rm Dir.glob("#{target_dir}/profile_*.tf.json")
+
+            account[:profiles].each do |profile|
+                profile_block = {
+                    'module' => {
+                        profile => {
+                            'source'    => "../../../profiles/#{profile}",
+                            'config'    => '${var.config}',
+                            'providers' => {
+                                'aws' => 'aws',
+                            },
+                        },
+                    },
+                }
+
+                write_json_block("#{target_dir}/profile_#{profile}.tf.json", profile_block)
+            end
+
+        end
+    end
+end

data/lib/tfctl/logger.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require 'logger'
+
+module Tfctl
+    class Logger
+
+        def initialize(log_level)
+            @outlog  = ::Logger.new(STDOUT)
+
+            self.level = log_level
+
+            @outlog.formatter = proc do |severity, _datetime, _progname, msg|
+                # "#{datetime.iso8601} #{severity.downcase}: #{msg}\n"
+                "#{severity.downcase}: #{msg}\n"
+            end
+        end
+
+        def level=(level)
+            @outlog.level = level
+        end
+
+        def level
+            @outlog.level
+        end
+
+        def debug(msg)
+            log(:debug, msg)
+        end
+
+        def info(msg)
+            log(:info, msg)
+        end
+
+        def warn(msg)
+            log(:warn, msg)
+        end
+
+        def error(msg)
+            log(:error, msg)
+        end
+
+        def fatal(msg)
+            log(:fatal, msg)
+        end
+
+        def log(level, msg)
+            @outlog.send(level, msg)
+        end
+
+    end
+end

data/lib/tfctl/schema.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require 'json_schemer'
+require_relative 'error.rb'
+
+# Config validator using JSON schema
+
+module Tfctl
+    module Schema
+        class << self
+
+            def validate(data)
+                schemer = JSONSchemer.schema(main_schema)
+                issues = []
+                schemer.validate(data).each do |issue|
+                    issues << {
+                        details:      issue['details'],
+                        data_pointer: issue['data_pointer'],
+                    }
+                end
+
+                return if issues.empty?
+
+                raise Tfctl::ValidationError.new('Config validation failed', issues)
+            end
+
+            private
+
+            def main_schema
+                iam_arn_pattern = 'arn:aws:iam:[a-z\-0-9]*:[0-9]{12}:[a-zA-Z\/+@=.,]*'
+
+                # rubocop:disable Layout/AlignHash
+                {
+                    'type'       => 'object',
+                    'properties' => {
+                        'tf_state_bucket'         => { 'type' => 'string' },
+                        'tf_state_role_arn'       => {
+                            'type'    => 'string',
+                            'pattern' => iam_arn_pattern,
+                        },
+                        'tf_state_dynamodb_table' => { 'type' => 'string' },
+                        'tf_state_region'         => { 'type' => 'string' },
+                        'tf_required_version'     => { 'type' => 'string' },
+                        'aws_provider_version'    => { 'type' => 'string' },
+                        'tfctl_role_arn'          => {
+                            'type'    => 'string',
+                            'pattern' => iam_arn_pattern,
+                        },
+                        'data'                    => { 'type' => 'object' },
+                        'exclude_accounts'        => { 'type' => 'array' },
+                        'organization_root'       => org_schema,
+                        'organization_units'      => org_schema,
+                        'account_overrides'       => org_schema,
+                    },
+                    'required'   => %w[
+                        tf_state_bucket
+                        tf_state_role_arn
+                        tf_state_dynamodb_table
+                        tf_state_region
+                        tfctl_role_arn
+                    ],
+                    'additionalProperties' => false,
+                }
+                # rubocop:enable Layout/AlignHash
+            end
+
+            def org_schema
+                {
+                    'type'       => 'object',
+                    'properties' => {
+                        'profiles'          => { 'type'=> 'array' },
+                        'data'              => { 'type'=> 'object' },
+                        'tf_execution_role' => { 'type'=> 'string' },
+                        'region'            => { 'type'=> 'string' },
+                    },
+                }
+            end
+        end
+    end
+end