tfctl 0.0.2

data/lib/tfctl/config.rb

@@ -0,0 +1,177 @@
+# frozen_string_literal: true
+
+require_relative '../hash.rb'
+require_relative 'error.rb'
+require 'yaml'
+require 'json'
+
+module Tfctl
+    class Config
+        include Enumerable
+        attr_reader :config
+
+        def initialize(config_name:, yaml_config:, aws_org_config:, use_cache: false)
+            cache_file  = "#{PROJECT_ROOT}/.tfctl/#{config_name}_cache.yaml"
+
+            # Get configuration.  Either load from cache or process fresh.
+            if use_cache
+                @config = read_cache(cache_file)
+            else
+                @config = load_config(config_name, yaml_config, aws_org_config)
+                write_cache(@config, cache_file)
+            end
+        end
+
+        def [](key)
+            @config[key]
+        end
+
+        def each(&block)
+            @config.each(&block)
+        end
+
+        def has_key?(k)
+            @config.has_key?(k)
+        end
+
+        def to_yaml
+            @config.to_yaml
+        end
+
+        def to_json
+            @config.to_json
+        end
+
+        # Filters accounts by account property
+        def find_accounts(property_name, property_value)
+            output =[]
+            @config[:accounts].each do |account|
+                if account[property_name] == property_value
+                    output << account
+                end
+            end
+
+            if output.empty?
+                raise Tfctl::Error.new "Account not found with #{property_name}: #{property_value}"
+            end
+            output
+        end
+
+        def find_accounts_regex(property_name, expr)
+            output =[]
+            @config[:accounts].each do |account|
+                begin
+                    if account[property_name] =~ /#{expr}/
+                        output << account
+                    end
+                rescue RegexpError => e
+                    raise Tfctl::Error.new "Regexp: #{e}"
+                end
+            end
+
+            if output.empty?
+                raise Tfctl::Error.new "Account not found with #{property_name} matching regex: #{expr}"
+            end
+            output
+        end
+
+
+        private
+
+        # Retrieves AWS Organizations data and merges it with data from yaml config.
+        def load_config(config_name, yaml_config, aws_org_config)
+
+            # AWS Organizations data
+            config = aws_org_config
+            # Merge organization sections from yaml file
+            config = merge_accounts_config(config, yaml_config)
+            # Import remaining parameters from yaml file
+            config = import_yaml_config(config, yaml_config)
+            # Set excluded property on any excluded accounts
+            config = mark_excluded_accounts(config)
+            # Remove any profiles that are unset
+            config = remove_unset_profiles(config)
+            # Set config name property (based on yaml config file name)
+            config[:config_name] = config_name
+            config
+        end
+
+        def write_cache(config, cache_file)
+            FileUtils.mkdir_p File.dirname(cache_file)
+            File.open(cache_file, 'w') {|f| f.write self.to_yaml }
+        end
+
+        def read_cache(cache_file)
+            unless File.exist?(cache_file)
+                raise Tfctl::Error.new("Cached configuration not found in: #{cache_file}")
+            end
+
+            YAML.load_file(cache_file)
+        end
+
+        # Sets :excluded property on any excluded accounts
+        def mark_excluded_accounts(config)
+          return config unless config.has_key?(:exclude_accounts)
+
+          config[:accounts].each_with_index do |account, idx|
+              if config[:exclude_accounts].include?(account[:name])
+                  config[:accounts][idx][:excluded] = true
+              else
+                  config[:accounts][idx][:excluded] = false
+              end
+          end
+          config
+        end
+
+        def remove_unset_profiles(config)
+            config[:accounts].each do |account|
+                profiles_to_unset = []
+                account[:profiles].each do |profile|
+                    if profile =~  /\.unset$/
+                        profiles_to_unset << profile
+                        profiles_to_unset << profile.chomp('.unset')
+                    end
+                end
+                account[:profiles] = account[:profiles] - profiles_to_unset
+            end
+            config
+        end
+
+        # Import yaml config other than organisation defaults sections which are merged elsewhere.
+        def import_yaml_config(config, yaml_config)
+            yaml_config.delete(:organization_root)
+            yaml_config.delete(:organization_units)
+            yaml_config.delete(:account_overrides)
+            config.merge(yaml_config)
+        end
+
+        # Merge AWS Organizations accounts config with defaults from yaml config
+        def merge_accounts_config(config, yaml_config)
+
+            config[:accounts].each_with_index do |account_config, idx|
+                account_name       = account_config[:name].to_sym
+                account_ou_parents = account_config[:ou_parents]
+
+                # merge any root settings
+                account_config = account_config.deep_merge(yaml_config[:organization_root])
+
+                # merge all OU levels settings
+                account_ou_parents.each_with_index do |_, i|
+                    account_ou = account_ou_parents[0..i].join('/').to_sym
+                    if yaml_config[:organization_units].has_key?(account_ou)
+                        account_config = account_config.deep_merge(yaml_config[:organization_units][account_ou])
+                    end
+                end
+
+                # merge any account overrides
+                if yaml_config[:account_overrides].has_key?(account_name)
+                    account_config = account_config.deep_merge(yaml_config[:account_overrides][account_name])
+                end
+
+                config[:accounts][idx] = account_config
+            end
+            config
+        end
+
+    end
+end

data/lib/tfctl/error.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Tfctl
+    class Error < StandardError
+    end
+end

data/lib/tfctl/executor.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+require 'open3'
+require 'fileutils'
+require 'shellwords'
+require 'thread'
+require_relative 'error.rb'
+
+module Tfctl
+    module Executor
+        extend self
+
+        # Execute terraform command
+        def run(account_name:, config_name:, log:, cmd: nil, argv: [], unbuffered: true)
+
+            if cmd.nil?
+                if File.exists?("#{PROJECT_ROOT}/bin/terraform")
+                    # use embedded terraform binary
+                    cmd = "#{PROJECT_ROOT}/bin/terraform"
+                else
+                    cmd = 'terraform'
+                end
+            end
+
+            path       = "#{PROJECT_ROOT}/.tfctl/#{config_name}/#{account_name}"
+            cwd        = FileUtils.pwd
+            plan_file  = "#{path}/tfplan"
+            semaphore  = Mutex.new
+            output     = []
+
+            # Extract terraform sub command from argument list
+            args       = Array.new(argv)
+            subcmd     = args[0]
+            args.delete_at(0)
+
+            # Enable plan file for `plan` and `apply` sub commands
+            args += plan_file_args(plan_file, subcmd)
+
+            # Create the command
+            exec = [cmd] + [subcmd] + args
+
+            # Set environment variables for Terraform
+            env = {
+                'TF_INPUT'           => '0',
+                'CHECKPOINT_DISABLE' => '1',
+                'TF_IN_AUTOMATION'   => 'true',
+                # 'TF_LOG'             => 'TRACE'
+            }
+
+            log.debug "#{account_name}: Executing: #{exec.shelljoin}"
+
+            FileUtils.cd path
+            Open3.popen3(env, exec.shelljoin) do |stdin, stdout, stderr, wait_thr|
+                stdin.close_write
+
+                # capture stdout and stderr in separate threads to prevent deadlocks
+                Thread.new do
+                    stdout.each do |line|
+                        semaphore.synchronize do
+                            unbuffered ? log.info("#{account_name}: #{line.chomp}") : output << [ 'info', line ]
+                        end
+                    end
+                end
+                Thread.new do
+                    stderr.each do |line|
+                        semaphore.synchronize do
+                            unbuffered ? log.error("#{account_name}: #{line.chomp}") : output << [ 'error', line ]
+                        end
+                    end
+                end
+
+                status = wait_thr.value
+
+                # log the output
+                output.each do |line|
+                    log.send(line[0], "#{account_name}: #{line[1].chomp}")
+                end
+
+                FileUtils.cd cwd
+                FileUtils.rm_f plan_file if args[0] == 'apply' # tidy up the plan file
+
+                unless status.exitstatus == 0
+                    raise Tfctl::Error.new "#{cmd} failed with exit code: #{status.exitstatus}"
+                end
+            end
+        end
+
+        # Adds plan file to `plan` and `apply` sub commands
+        def plan_file_args(plan_file, subcmd)
+            output = []
+            if subcmd == 'plan'
+                output = [ "-out=#{plan_file}" ]
+
+            elsif subcmd == 'apply'
+                if File.exists?(plan_file)
+                    output = [ "#{plan_file}" ]
+                else
+                    raise Tfctl::Error.new "Plan file not found in #{plan_file}.  Run plan first."
+                end
+            end
+            output
+        end
+    end
+end

data/lib/tfctl/generator.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+
+# Generates Terraform configuration for an account.
+
+module Tfctl
+    module Generator
+        extend self
+
+        def write_json_block(path, block)
+            File.open(path, 'w') do |f|
+                f.write(JSON.pretty_generate(block) + "\n")
+            end
+        end
+
+        def make(
+            terraform_io_org:,
+            account_id:,
+            account_name:,
+            execution_role:,
+            profiles:,
+            config:,
+            region: 'eu-west-1',
+            tf_version: '>= 0.12.0',
+            aws_provider_version: '~> 2.14',
+            target_dir: "#{PROJECT_ROOT}/.tfctl/#{config[:config_name]}/#{account_name}"
+        )
+
+            FileUtils.mkdir_p target_dir
+
+            terraform_block = {
+                'terraform' => {
+                    'required_version' => tf_version,
+                    'backend' => {
+                        's3' => {
+                            'bucket'         => config[:tf_state_bucket],
+                            'key'            => "#{account_name}/tfstate",
+                            'region'         => config[:tf_state_region],
+                            'role_arn'       => config[:tf_state_role_arn],
+                            'dynamodb_table' => config[:tf_state_dynamodb_table],
+                            'encrypt'        => 'true',
+                        }
+                    }
+                }
+            }
+            write_json_block("#{target_dir}/terraform.tf.json", terraform_block)
+
+            provider_block = {
+                'provider' => {
+                    'aws'  => {
+                        'version'     => aws_provider_version,
+                        'region'      => region,
+                        'assume_role' => {
+                            'role_arn' => "arn:aws:iam::#{account_id}:role/#{execution_role}"
+                        }
+                    }
+                }
+            }
+            write_json_block("#{target_dir}/provider.tf.json", provider_block)
+
+            vars_block = {
+                'variable' => {
+                    'config' => {
+                        'type' => 'string'
+                    }
+                }
+            }
+            write_json_block("#{target_dir}/vars.tf.json", vars_block)
+
+            # config is passed to profiles as a json encoded string.  It can be
+            # decoded in profile using jsondecode() function.
+            config_block = { 'config' => config.to_json }
+            write_json_block("#{target_dir}/config.auto.tfvars.json", config_block)
+
+            FileUtils.rm Dir.glob("#{target_dir}/profile_*.tf.json")
+
+            profiles.each do |profile|
+                profile_block = {
+                    'module' => {
+                        profile => {
+                            'source' => "../../../profiles/#{profile}",
+                            'config' => '${var.config}',
+                            'providers' => {
+                              'aws' => 'aws'
+                            }
+                        }
+                    }
+                }
+
+                write_json_block("#{target_dir}/profile_#{profile}.tf.json", profile_block)
+            end
+
+        end
+    end
+end

data/lib/tfctl/logger.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require 'logger'
+
+module Tfctl
+    class Logger
+
+        def initialize(log_level)
+            @outlog  = ::Logger.new(STDOUT)
+
+            self.level = log_level
+
+            @outlog.formatter = proc do |severity, datetime, progname, msg|
+                # "#{datetime.iso8601} #{severity.downcase}: #{msg}\n"
+                "#{severity.downcase}: #{msg}\n"
+            end
+        end
+
+        def level=(level)
+            @outlog.level = level
+        end
+
+        def level
+            @outlog.level
+        end
+
+        def debug(msg); log(:debug, msg); end
+        def info(msg); log(:info, msg); end
+        def warn(msg); log(:warn, msg); end
+        def error(msg); log(:error, msg); end
+        def fatal(msg); log(:fatal, msg); end
+
+        def log(level, msg)
+            @outlog.send(level, msg)
+        end
+
+    end
+end

data/lib/tfctl/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Tfctl
+    VERSION = '0.0.2'
+end

data/lib/tfctl.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require_relative 'tfctl/config.rb'
+require_relative 'tfctl/generator.rb'
+require_relative 'tfctl/executor.rb'
+require_relative 'tfctl/error.rb'
+require_relative 'tfctl/logger.rb'
+require_relative 'tfctl/aws_org.rb'
+require_relative 'tfctl/version.rb'

data/tfctl.gemspec

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+$LOAD_PATH << File.expand_path("../lib", __FILE__)
+require 'tfctl/version'
+
+Gem::Specification.new do |spec|
+    spec.name          = 'tfctl'
+    spec.version       = Tfctl::VERSION
+    spec.authors       = [
+        'Andrew Wasilczuk'
+    ]
+    spec.email         = [
+        'akw@scalefactory.com'
+    ]
+    spec.summary       = 'Terraform wrapper for managing multi-account AWS infrastructures'
+    spec.homepage      = 'https://github.com/scalefactory/tfctl'
+    spec.license       = "MIT"
+    spec.files         = `git ls-files -z`.split("\x0").reject { |f|
+        f.match(%r{^(test|spec|features)/})
+    }
+    spec.bindir        = "bin"
+    spec.executables   = spec.files.grep(%r{^bin/tfctl}) { |f| File.basename(f) }
+    spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+    spec.require_paths = ["lib"]
+
+    spec.add_dependency 'aws-sdk-organizations', '~> 1.13'
+    spec.add_dependency 'parallel',              '~> 1.17'
+
+    spec.add_development_dependency 'rspec', '~> 3.8'
+end

metadata

@@ -0,0 +1,119 @@
+--- !ruby/object:Gem::Specification
+name: tfctl
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- Andrew Wasilczuk
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-11-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-organizations
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.17'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.17'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.8'
+description: 
+email:
+- akw@scalefactory.com
+executables:
+- tfctl
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- CHANGELOG.adoc
+- Gemfile
+- LICENSE
+- Makefile
+- README.adoc
+- bin/tfctl
+- docs/configuration.adoc
+- docs/control_tower.adoc
+- docs/creating_a_profile.adoc
+- docs/iam_permissions.adoc
+- docs/project_layout.adoc
+- examples/bootstrap/terraform-exec-role.template
+- examples/bootstrap/terraform-state.template
+- examples/bootstrap/tfctl-org-access.template
+- examples/control_tower/conf/example.yaml
+- examples/control_tower/modules/s3-bucket/main.tf
+- examples/control_tower/modules/s3-bucket/variables.tf
+- examples/control_tower/profiles/example-profile/data.tf
+- examples/control_tower/profiles/example-profile/main.tf
+- examples/control_tower/profiles/example-profile/variables.tf
+- lib/hash.rb
+- lib/tfctl.rb
+- lib/tfctl/aws_org.rb
+- lib/tfctl/config.rb
+- lib/tfctl/error.rb
+- lib/tfctl/executor.rb
+- lib/tfctl/generator.rb
+- lib/tfctl/logger.rb
+- lib/tfctl/version.rb
+- tfctl.gemspec
+homepage: https://github.com/scalefactory/tfctl
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: Terraform wrapper for managing multi-account AWS infrastructures
+test_files: []