tfctl 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 0e1075714676ca848752725f2d98c38240e5a9ee2bc901b582ffba6edd46fc0f
+  data.tar.gz: b48bc64d5a9aaae352538ed88b648b20e2b2c68c59bd0c1b3b5c8757e3c421b9
+SHA512:
+  metadata.gz: b87ecf1c66f1528bf98134d631aef3c8264326bbd0e399eb769860ad853d24ebc59339ba6df16be41b6d728c4fd63df035a210a1ca9d09fd17960de2b4d66fa1
+  data.tar.gz: 657f44ce06c1ac667525a80009f73e88fbf9754f32f5703faad8fbfe1b6355fd6796690c8b0583af549fef8d75de318d1dc98f56aa80644b78ff2a068e30bfa7

data/.gitignore

@@ -0,0 +1,13 @@
+.DS_Store
+*.swp
+.tfctl
+pkg/
+*.gem
+vendor/
+.bundle
+bin/bundle
+bin/htmldiff
+bin/ldiff
+bin/rspec
+spec/reports
+Gemfile.lock

data/.rspec

@@ -0,0 +1 @@
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,17 @@
+rvm:
+- 2.3
+- 2.6
+sudo: false
+script: make test
+deploy:
+  provider: rubygems
+  api_key:
+    secure: FKAONS7x6koN7oiEULr4ViwjlDBzbE0bCgqhXRP4DfTtlRyEymeqgrfSIqkVH7unjd0muIGrMBnFuaQVSx7648RS1Ss0QAJo32SVnzoYl1P03cijqNmbbaf1jRdA3IGmh0gV5vsXrmlHiP8gfAuC9PqQ550OzxzWUvEI8vXgSTibmKd/PoQinv5g/dq0gBFjlhSMt/k3Z9WlMmkEsAro/r/Ie2M7mItHPT65f0ga5q5SeujPQQ3Sd/l3mznh37bmnw5RZpFDYdA7jL2p0Y58XJPBU8soa3ZC5GeHyxCYVoGh6EDGAFb83ERRT6rQ7ywkOufTv1o497P7a/prSbvT6fzc+DcugXPEaglT+dUXMe36OoF907Xva4vq3xIHV2N/yrxbDM85hmMk22wEU+9wpDDzFNQnfsXNbaHG9F7gLgy0eoTRrSuJf6cPDlE8pwvn7b8cjieeqWc//ZNhSYnHYZGER4LFINWVxs68Eofmmqp2IESTcUpJ8oB4bV+bzzyobJMRobOXu2hvgCrTdr6r/PnckpAfZE/l4nVQa14f1FU//8bU3DwvNun6TX1Ujp+XNiRDUlvP2KnkBU4s5rsIkL3lCHW7r6GipSk6SOvGMTz5eySMsoWvZQBdAzk/OxcIteeWH9pdo1Hbu5x2/bwyuTRCQ9E79CKWDKlIQwCgUY0=
+  gem: tfctl
+  on:
+    tags: true
+    repo: scalefactory/tfctl
+notifications:
+  slack:
+    rooms:
+      secure: HrdsUaTCvwtq5rLyAf7Uwo76p9JhxnoFvoZU/AtxTCL6lwMm5pqpfDAdbBH8C2BYMWRyiS8IfSrvuLbGUr317qJgVCmnhD7JIgdA0Aih4GkR0yK+EVao7qdOClO2CGJFJGqHveBBpXy94FEwgFwQSvnuSZdY8gNdXafvpgrPvcbWHIF9Bq7StfYelWZX6+FL7xczqVhdetXcHJEj6Q/thkbvyn0hJAVegjDq4Pw3MyR66Mv5HTj+jYbGrJOR1poNW8vv4iQjK0TIMvCCeTv4Ddrjitn1kOK1BPYqoHEkaRwXdS7+hevv+UfV074awTp3UOIS5VP3ve/KX21xcl9DWMG7Gde9tuyap9WKhN3XE5686qrfn4M/L7PmVnzeYimUKLQfdqjDXa7UI3orGIOq8nKCsCUFvFUQZ/DxeEssJzBNNRbeGywwVtXk7/8L2HHiQJ3t/29vMLEq6EwskYK3Uao2aIVWpLSkPwOuwVxrNWXrwojrrKo+oo8QvW6xkov5UW3VkapyzRyDN2oIpNkFeLYfuyTO4MBsnLzCBqJYEeku2l0Hp9WBq2Q1a+FRgqQlcRbLPsYwK5AdkRKcWuXP8FMZoqXX6F9B73q5Dbub2et4YN4zhvroLZN6MmlneM6faJh9aS4yV2ZlU1XbOtbqP0nkHrJ+o8DeM/wgTB1GB90=

data/CHANGELOG.adoc

@@ -0,0 +1,9 @@
+= Changelog
+
+== 0.0.2
+
+* BUGFIX: Fixed an exception when `exclude_accounts` is not set.
+
+== 0.0.1
+
+* Initial release

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2019 Essentia Analytics Ltd
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/Makefile

@@ -0,0 +1,26 @@
+.PHONY: clean install test
+
+vendor:
+	$(info => Installing Ruby dependencies)
+	@bundle install --path vendor --with developement --binstubs=vendor/bin
+
+test: vendor
+	$(info => Running spec tests)
+	@vendor/bin/rspec
+
+pkg:
+	$(info => Building gem package in pkg/)
+	@mkdir pkg/
+	@gem build tfctl.gemspec
+	@mv *.gem pkg/
+
+install: pkg
+	gem install pkg/*.gem
+
+clean:
+	$(info => Cleaning)
+	@rm -rf pkg/
+	@rm -rf vendor/
+	@rm -rf .bundle
+	@rm -f Gemfile.lock
+	@rm -rf spec/reports/

data/README.adoc

@@ -0,0 +1,145 @@
+:toc:
+
+= tfctl
+
+== Overview
+
+Tfctl is a small Terraform wrapper for working with multi-account AWS
+infrastructures where new accounts may be created dynamically and on-demand.
+
+Discovers accounts by reading the AWS Organizations API and can assign
+Terraform resources to accounts based on the organization hierarchy.  Resources
+can be assigned globally, based on organization unit or individual accounts.
+It supports nested OU hierarchies.
+
+Tfctl was originally developed to integrate Terraform with
+https://aws.amazon.com/solutions/aws-landing-zone/[AWS Landing Zone] and
+https://aws.amazon.com/controltower/[Control Tower] but should work with most
+other ways of managing accounts in AWS Organizations.
+
+== Features
+
+* Discovers AWS accounts automatically.
+* Automatically generates Terraform account configuration.
+* Parallel execution across multiple accounts.
+* Hierarchical configuration based on AWS Organization units structure.
+* Supports per account configuration overrides for handling exceptions.
+* Supports nested organization units.
+* Terraform state tracking in S3 and locking in DynamoDB.
+* Account targeting by OU path regular expressions.
+* Automatic role assumption in target accounts.
+* Works with CI/CD pipelines.
+
+== Requirements
+
+ * Terraform >= 0.12
+ * Ruby >= 2.3
+ * Accounts managed in AWS Organizations (by Landing Zone, Control Tower, some
+   other means)
+
+== Installation
+
+Clone this repository and run:
+
+----
+make install
+----
+
+This will build a ruby gem and install it.
+
+When using bundler, add this to your `Gemfile`:
+
+----
+gem 'tfctl', git: 'https://github.com/scalefactory/tfctl'
+----
+
+== Docs
+
+ * https://github.com/scalefactory/tfctl/tree/master/docs/control_tower.adoc[Control Tower quick start guide]
+ * https://github.com/scalefactory/tfctl/tree/master/docs/project_layout.adoc[Project layout]
+ * https://github.com/scalefactory/tfctl/tree/master/docs/configuration.adoc[Configuration]
+ * https://github.com/scalefactory/tfctl/tree/master/docs/iam_permissions.adoc[IAM permissions]
+ * https://github.com/scalefactory/tfctl/tree/master/docs/creating_a_profile.adoc[Creating a profile]
+
+== Running tfctl
+
+tfctl should be run from the root of the project directory.  It will generate
+Terraform configuration in `.tfctl/`.
+
+Anatomy of a tfctl command:
+
+----
+tfctl -c CONFIG_FILE TARGET_OPTIONS -- TERRAFORM_COMMAND
+----
+
+* `-c` specifies which tfctl config file to use (usually in `conf/`)
+* `TARGET_OPTIONS` specifies which accounts to target.  This could be an individual
+  account, a group of accounts in an organizational unit or all accounts.
+* `TERRAFORM_COMMAND` will be passed to `terraform` along with any
+  options.  See https://www.terraform.io/docs/commands/index.html[Terraform
+  commands] for details.
+
+NOTE: You must have your AWS credentials configured before running tfctl or run
+it using an AWS credentials helper such as
+https://github.com/99designs/aws-vault[aws-vault].
+
+=== Example commands
+
+Show help:
+
+----
+tfctl -h
+----
+
+Show merged configuration:
+
+----
+tfctl -c conf/example.yaml -s
+----
+
+Run Terraform init accross all accounts:
+
+----
+tfctl -c conf/example.yaml --all -- init
+----
+
+Run plan in `test` OU accounts:
+
+----
+tfctl -c conf/example.yaml -o test -- plan
+----
+
+Run plan in `live` accounts assuming that `live` is a child OU in multiple
+organization units:
+
+----
+tfctl -c conf/example.yaml -o '.*/live' -- plan
+----
+
+Run plan in an individual account:
+
+----
+tfctl -c conf/example.yaml -a example-account - plan
+----
+
+Run apply in all accounts:
+
+----
+tfctl -c conf/example.yaml --all -- apply
+----
+
+Run destroy in `test` OU accounts:
+
+----
+tfctl -c conf/example.yaml -o test -- destroy -auto-approve
+----
+
+Don't buffer the output:
+
+----
+tfctl -c conf/example.yaml -a example-account -u -- plan
+----
+
+This will show output in real time.  Usually output is buffered and displayed
+after Terraform command finishes to make it more readable when running across
+multiple accounts in parallel.

data/bin/tfctl

@@ -0,0 +1,198 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+if File.directory?(File.dirname(__FILE__) + '/../vendor')
+    require 'bundler/setup'
+end
+require 'optparse'
+require 'fileutils'
+require 'parallel'
+require_relative '../lib/tfctl'
+
+PROJECT_ROOT = Dir.pwd
+
+#
+# Process CLI arguments
+#
+
+options={
+    :account     => nil,
+    :ou          => nil,
+    :all         => nil,
+    :show_config => false,
+    :config_file => nil,
+    :unbuffered  => false,
+    :debug       => false,
+    :use_cache   => false
+}
+
+optparse = OptionParser.new do |opts|
+    opts.on('-a', '--account=name', "Target a specific AWS account") do |o|
+        options[:account] = o
+    end
+    opts.on('-o', '--ou=organization_unit', "Target accounts in an Organization Unit (uses regex matching)") do |o|
+        options[:ou] = o
+    end
+    opts.on('--all', "Target all accounts") do
+        options[:all] = true
+    end
+    opts.on('-c', '--config-file=config', "Path to config file") do |o|
+        options[:config_file] = o
+    end
+    opts.on('-s', '--show-config', "Display configuration") do
+        options[:show_config] = true
+    end
+    opts.on('-x', '--use-cache', "Use cached AWS organization data") do
+        options[:use_cache] = true
+    end
+    opts.on('-u', '--unbuffered', "Disable buffering of Terraform output") do
+        options[:unbuffered] = true
+    end
+    opts.on('-d', '--debug', "Turn on debug messages") do
+        options[:debug] = true
+    end
+    opts.on('-v', '--version', "Show version") do
+        puts Tfctl::VERSION
+        exit
+    end
+end
+
+
+begin
+    optparse.parse!
+
+    # Validate CLI arguments
+
+    if options[:config_file].nil?
+        raise OptionParser::MissingArgument, '--config-file'
+    end
+
+    unless File.exists? options[:config_file]
+        raise OptionParser::InvalidOption,
+          "Config file not found in: #{options[:config_file]}"
+    end
+
+    unless File.exists? options[:config_file]
+      raise OptionParser::InvalidOption, "Config file #{options[:config_file]} not found."
+    end
+
+    # Validate targets
+    targetting_opts = [:account, :ou, :all]
+    targets_set = []
+    options.each do |k,v|
+        if targetting_opts.include?(k)
+            targets_set << k.to_s unless v == nil
+        end
+    end
+    if targets_set.length > 1
+        raise OptionParser::InvalidOption,
+            "Too many target options set: #{targets_set.join(', ')}.  Only one can be specified."
+    end
+    if targets_set.empty? and options[:show_config] == false
+        raise OptionParser::InvalidOption, "Please specify target"
+    end
+
+rescue OptionParser::InvalidOption, OptionParser::MissingArgument
+    $stderr.puts $!.to_s
+    $stderr.puts optparse
+    exit 2
+end
+
+
+
+# Generates configuration and runs Terraform commands for a target account.
+def run_account(config, account, options, tf_argv, log)
+
+    # Skip excluded accounts
+    if account[:excluded] == true
+        log.info "#{account[:name]}: excluded, skipping"
+        return
+    end
+
+    # Generate Terraform run directory with configured providers, backend and
+    # profiles for the target account.  This is where Terraform will be
+    # executed from.
+    log.info "#{account[:name]}: Generating Terraform run directory"
+    Tfctl::Generator.make(
+        config:            config,
+        terraform_io_org:  config[:terraform_io_org],
+        account_id:        account[:id],
+        account_name:      account[:name],
+        profiles:          account[:profiles],
+        execution_role:    account[:tf_execution_role]
+    )
+
+    log.info "#{account[:name]}: Executing Terraform #{tf_argv[0]}"
+    Tfctl::Executor.run(
+        account_name: account[:name],
+        config_name:  config[:config_name],
+        unbuffered:   options[:unbuffered],
+        log:          log,
+        argv:         tf_argv
+    )
+end
+
+
+#
+# Main
+#
+
+begin
+
+    # Set up logging
+    options[:debug] ? log_level = Logger::DEBUG : log_level = Logger::INFO
+    log = Tfctl::Logger.new(log_level)
+
+    log.info 'tfctl running'
+
+    config_name = File.basename(options[:config_file]).chomp('.yaml')
+    log.info "Using config: #{config_name}"
+
+    log.info 'Working out AWS account topology'
+
+    yaml_config = YAML.load(File.read(options[:config_file]))
+    yaml_config.symbolize_names!
+
+    org_units = yaml_config[:organization_units].keys
+    aws_org_accounts = Tfctl::AwsOrg.new(yaml_config[:tfctl_role_arn]).accounts(org_units)
+
+    log.info 'Merging configuration'
+
+    config = Tfctl::Config.new(
+                config_name:    config_name,
+                yaml_config:    yaml_config,
+                aws_org_config: aws_org_accounts,
+                use_cache:      options[:use_cache])
+
+
+    if options[:show_config]
+        puts config.to_yaml
+        exit 0
+    end
+
+    # Run targets
+
+    if options[:account]
+        account = config.find_accounts(:name, options[:account]).first
+        run_account(config, account, options, ARGV, log)
+
+    elsif options[:ou]
+        accounts = config.find_accounts_regex(:ou_path, options[:ou])
+        Parallel.each(accounts, in_processes: 8) do |ac|
+            run_account(config, ac, options, ARGV, log)
+        end
+
+    elsif options[:all]
+        accounts = config[:accounts]
+        Parallel.each(accounts, in_processes: 8) do |ac|
+            run_account(config, ac, options, ARGV, log)
+        end
+    end
+
+
+    log.info 'Done'
+
+rescue Tfctl::Error => e
+    log.error(e)
+    exit 1
+end

data/docs/configuration.adoc

@@ -0,0 +1,53 @@
+== Configuration
+
+Tfctl retrieves initial account configuration from AWS Organizations and merges
+it with organization config specified in the yaml file.
+
+The configuration is merged in the following order:
+
+* AWS Organizations data is fetched and stored in an `accounts` array.
+* `organization_root` settings are merged with all accounts.
+* `organization_units` settings are merged with accounts matching the OU.
+* `account_overrides` are merged with individual accounts matching the account name.
+
+Parameters further down the hierarchy take precedence.  For example:
+
+[source, yaml]
+----
+organization_root:
+  example_param: 'will be overriden further down'
+
+organization_units:
+  team:
+    example_param: 'will win in team ou'
+  team/live:
+    example_param: 'will win in team/live ou'
+----
+
+One exception to this rule is the `profiles` parameter.  Profiles are additive:
+
+[source, yaml]
+----
+organization_root:
+  profiles:
+   - profile-one
+   - profile-two
+
+organization_units:
+  team:
+    profiles:
+      - profile-three
+----
+
+This will result in all three profiles deployed to accounts in `team` OU.
+
+TIP: You can display the fully merged configuration by running `tfctl -c
+conf/CONFIG_FILE.yaml -s`.  It's safe to run as it doesn't make any changes to
+AWS resources.  It's a good way to test your configuration.
+
+=== Handling secrets
+
+No secrets should be committed into Terraform or tfctl configuration.  Use AWS
+Secrets Manager instead and retrieve in Terraform profiles using
+https://www.terraform.io/docs/providers/aws/d/secretsmanager_secret.html[secrets
+manager data source]

data/docs/control_tower.adoc

@@ -0,0 +1,191 @@
+:toc:
+
+== Control Tower integration guide
+
+This guide will help you integrate Terraform with AWS Control Tower using the
+tfctl wrapper.  This involves setting up resources for remote state tracking,
+necessary IAM roles and a tfctl project.
+
+=== Overview
+
+For state tracking we're going to create a dedicated `shared-services` account
+under a `mgmt` organization unit.  We'll use S3 for state storage and DynamoDB
+for locking.  `TerraformState` IAM role will be created for cross account
+access to state resources from the primary account.
+
+In the primary account we'll create a `TfctlOrgAccess` role.  It gives tfctl
+read only access to AWS Organizations which is used to discover accounts and
+the organization unit structure.
+
+We'll use CloudFormation stacks and stack-sets to bootstrap these resources.
+
+For executing Terraform in spoke accounts we'll use the
+`AWSControlTowerExecution` role which is automatically created by Control Tower
+account factory and can be assumed from the primary account.
+
+We're going to create a `live` and `test` organization units in Control Tower
+and provision a couple of accounts for testing.
+
+=== Prerequisites
+
+Before starting you'll need:
+
+ * Control Tower set up in your primary account.
+ * A user with `AdministratorAccess` privileges in primary account.
+ * AWS CLI tools installed on your machine.
+ * Terraform 0.12 or higher.
+
+=== Configure Control Tower
+
+Create the following organization units in Control Tower:
+
+ * `mgmt`
+ * `live`
+ * `test`
+
+Then provision accounts:
+
+ * In `mgmt` OU create an account called `mgmt-shared-services`
+ * In `live` create `live-example1`
+ * In `test` create `test-example1`
+
+NOTE: Control Tower accounts need to be provisioned one at a time.  It takes
+approximately 20 mins to provision one.
+
+=== Install tfctl
+
+----
+git clone git@github.com:scalefactory/tfctl.git
+cd tfctl/ && sudo make install
+----
+
+=== Set up AWS resources
+
+It's assumed you have configured AWS CLI access to the primary account.
+
+We'll use CloudFormation templates in `examples/bootstrap/`.
+
+First export configuration using environment variables making sure to change to
+values to suit your set up:
+
+----
+export PRIMARY_ACCOUNT_ID=11111111
+export SHARED_SERVICES_ACCOUNT_ID=22222222
+export STATE_BUCKET_NAME='example-terraform-state'
+----
+
+Create the remote state resources stack set:
+
+----
+cd examples/bootstrap/
+
+aws cloudformation create-stack-set \
+    --stack-set-name TerraformState \
+    --template-body file://terraform-state.template \
+    --description "Resources for managing Terraform state" \
+    --capabilities CAPABILITY_NAMED_IAM CAPABILITY_IAM \
+    --execution-role-name AWSControlTowerExecution \
+    --administration-role-arn arn:aws:iam::${PRIMARY_ACCOUNT_ID}:role/service-role/AWSControlTowerStackSetRole \
+    --parameters ParameterKey=PrimaryAccountId,ParameterValue=${PRIMARY_ACCOUNT_ID} \
+    ParameterKey=TerraformStateBucket,ParameterValue=${STATE_BUCKET_NAME}
+----
+
+Create a stack set instance in you shared services account:
+
+----
+aws cloudformation create-stack-instances \
+    --stack-set-name TerraformState \
+    --accounts ${SHARED_SERVICES_ACCOUNT_ID} \
+    --regions eu-west-1
+----
+
+Check status:
+
+----
+aws cloudformation describe-stack-instance \
+    --stack-set-name TerraformState \
+    --stack-instance-account ${SHARED_SERVICES_ACCOUNT_ID} \
+    --stack-instance-region eu-west-1
+----
+
+NOTE: Initial status will be `OUTDATED`, it should change to `CURRENT` once deployed.
+
+Deploy `TfctlOrgAccess` IAM role stack:
+
+----
+aws cloudformation create-stack \
+    --stack-name TfctlOrgAccess \
+    --template-body file://tfctl-org-access.template \
+    --capabilities CAPABILITY_NAMED_IAM CAPABILITY_IAM \
+    --parameters ParameterKey=PrimaryAccountId,ParameterValue=${PRIMARY_ACCOUNT_ID}
+----
+
+Check status:
+
+----
+aws cloudformation describe-stacks --stack-name TfctlOrgAccess
+----
+
+NOTE: Successful status should read: `CREATE_COMPLETE`.
+
+=== Configure tfctl
+
+Copy the example project directory `examples/control_tower` somewhere convenient
+and edit `conf/example.yaml`.
+
+You need to modify the following parameters:
+
+ * `tf_state_bucket` - set to `$STATE_BUCKET_NAME`
+ * `tf_state_role_arn` - set shared services account ID
+ * `tfctl_role_arn` - set primary account ID
+ * `primary_account` - set the primary account name.  You can find it in AWS Organizations.
+
+TIP: You should keep your project directory under version control.
+
+=== Deploy example tfctl profile
+
+The example profile will create an S3 bucket in accounts under `test`, `live`
+and `mgmt` OUs.
+
+NOTE: Run tfctl commands from the root of you project directory.
+
+First dump the configuration to verify everything works:
+
+----
+tfctl -c conf/example.yaml -s
+----
+
+This will not make any changes but will print out a yaml containing the final,
+merged configuration data.  It should contain a list of discovered accounts and
+their configuration.
+
+Initialise terraform for all discovered accounts:
+
+----
+tfctl -c conf/example.yaml --all -- init
+----
+
+Tfctl will run Terraform against all accounts in parallel.
+
+Run plan:
+
+----
+tfctl -c conf/example.yaml --all -- plan
+----
+
+and apply:
+
+----
+tfctl -c conf/example.yaml --all -- apply
+----
+
+To destroy created resources run:
+
+----
+tfctl -c conf/example.yaml --all -- destroy -auto-approve
+----
+
+That's it! You can now execute terraform across your Control Tower estate.
+
+TIP: Your project directory should be under version control excluding the
+`.tfctl` directory which is automatically generated.