tfctl 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 6d47682cf9949db840c18d07b6f06907a9d36e2a75d0a5255b4c57f3603c0dbf
+  data.tar.gz: d5a81e877943fff53e903a104249e1147a827c593430bc91c0b78d5cc467fc4b
+SHA512:
+  metadata.gz: 03b7d69b7a7bbf296b0b1ab3aa8794d8b91e51559a1e2ee1d450396615aeba495b63993f84866c6fe5997d34c760da143cb66ca0cabc2a1a5a1266b701676bac
+  data.tar.gz: f9ebdae72fd58473c3a8cc5015fd2d2e32092488218b52efa79e2d625802ccdf4447298e45497d4ec0e03583b0edbd3a31b51e92f76a5981a5f3819f9cee1e4a

data/.gitignore

@@ -0,0 +1,13 @@
+.DS_Store
+*.swp
+.tfctl
+pkg/
+*.gem
+vendor/
+.bundle
+bin/bundle
+bin/htmldiff
+bin/ldiff
+bin/rspec
+spec/reports
+Gemfile.lock

data/.rspec

@@ -0,0 +1 @@
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,79 @@
+---
+AllCops:
+  TargetRubyVersion: 2.3
+  DisplayCopNames: true
+
+Layout/IndentationWidth:
+  Width: 4
+
+Layout/IndentHeredoc:
+  Enabled: false
+
+Layout/EmptyLines:
+  Enabled: false
+
+Layout/EmptyLinesAroundMethodBody:
+  Enabled: false
+
+Layout/AlignHash:
+  EnforcedHashRocketStyle:
+    - table
+  EnforcedColonStyle:
+    - table
+
+Layout/SpaceAroundOperators:
+  Enabled: false
+
+Layout/ExtraSpacing:
+  Enabled: false
+
+Layout/EmptyLinesAroundBlockBody:
+  Enabled: false
+
+Layout/EmptyLinesAroundClassBody:
+  Enabled: false
+
+Metrics/CyclomaticComplexity:
+  Enabled: false
+
+Metrics/PerceivedComplexity:
+  Enabled: false
+
+Metrics/BlockLength:
+  Enabled: false
+
+Metrics/MethodLength:
+  Enabled: false
+
+Metrics/LineLength:
+  Max: 140
+
+Metrics/AbcSize:
+  Enabled: false
+
+Metrics/ParameterLists:
+  Enabled: false
+
+Metrics/ClassLength:
+  Enabled: false
+
+Style/IfUnlessModifier:
+  Enabled: false
+
+Style/AndOr:
+  Enabled: false
+
+Style/Documentation:
+  Enabled: false
+
+Style/TrailingCommaInArguments:
+    EnforcedStyleForMultiline: comma
+
+Style/TrailingCommaInArrayLiteral:
+    EnforcedStyleForMultiline: comma
+
+Style/TrailingCommaInHashLiteral:
+    EnforcedStyleForMultiline: comma
+
+Style/RedundantReturn:
+  Enabled: false

data/.travis.yml

@@ -0,0 +1,18 @@
+rvm:
+  - 2.3
+  - 2.6
+os: linux
+language: ruby
+script: make test
+jobs:
+  include:
+    - stage: Gem release
+      rvm: 2.6
+      deploy:
+        provider: rubygems
+        api_key:
+          secure: FKAONS7x6koN7oiEULr4ViwjlDBzbE0bCgqhXRP4DfTtlRyEymeqgrfSIqkVH7unjd0muIGrMBnFuaQVSx7648RS1Ss0QAJo32SVnzoYl1P03cijqNmbbaf1jRdA3IGmh0gV5vsXrmlHiP8gfAuC9PqQ550OzxzWUvEI8vXgSTibmKd/PoQinv5g/dq0gBFjlhSMt/k3Z9WlMmkEsAro/r/Ie2M7mItHPT65f0ga5q5SeujPQQ3Sd/l3mznh37bmnw5RZpFDYdA7jL2p0Y58XJPBU8soa3ZC5GeHyxCYVoGh6EDGAFb83ERRT6rQ7ywkOufTv1o497P7a/prSbvT6fzc+DcugXPEaglT+dUXMe36OoF907Xva4vq3xIHV2N/yrxbDM85hmMk22wEU+9wpDDzFNQnfsXNbaHG9F7gLgy0eoTRrSuJf6cPDlE8pwvn7b8cjieeqWc//ZNhSYnHYZGER4LFINWVxs68Eofmmqp2IESTcUpJ8oB4bV+bzzyobJMRobOXu2hvgCrTdr6r/PnckpAfZE/l4nVQa14f1FU//8bU3DwvNun6TX1Ujp+XNiRDUlvP2KnkBU4s5rsIkL3lCHW7r6GipSk6SOvGMTz5eySMsoWvZQBdAzk/OxcIteeWH9pdo1Hbu5x2/bwyuTRCQ9E79CKWDKlIQwCgUY0=
+        gem: tfctl
+        on:
+          tags: true
+          repo: scalefactory/tfctl

data/CHANGELOG.adoc

@@ -0,0 +1,29 @@
+= Changelog
+
+== 1.0.0
+
+* feat(config): JSON schema config validation
+* feat(config): added 'data' parameter
+
+BREAKING CHANGE: This release moves user defined data under a separate `data`
+parameter so it can be easily distinguished from parameters required by tfctl.
+Configuration file will need to be updated to reflect this to pass validation.
+
+
+== 0.2.0
+
+* feat: configurable Terraform and AWS provider version requirements
+* fix: use provider region from config file
+* fix: fail when terraform command is missing
+
+== 0.1.0
+
+* feat: Added `-l` switch to list discovered accounts.
+
+== 0.0.2
+
+* fix: Fixed an exception when `exclude_accounts` is not set.
+
+== 0.0.1
+
+* Initial release

data/Gemfile

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+gemspec

data/Guardfile

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+guard :rspec, cmd: 'bundle exec rspec' do
+    watch(%r{^spec/.+_spec\.rb$})
+    watch(%r{^lib/tfctl/(.+)\.rb$}) { |m| "spec/#{m[1]}_spec.rb" }
+    watch('spec/spec_helper.rb')  { 'spec' }
+end

data/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2019 Essentia Analytics Ltd
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/Makefile

@@ -0,0 +1,36 @@
+.PHONY: clean install test rubocop spec guard
+
+vendor:
+	$(info => Installing Ruby dependencies)
+	@bundle install --path vendor --with developement --binstubs=vendor/bin
+
+test: vendor rubocop spec
+
+guard: vendor
+	$(info => Starting guard)
+	@bundle exec guard
+
+rubocop:
+	$(info => Running rubocop)
+	@vendor/bin/rubocop
+
+spec:
+	$(info => Running spec tests)
+	@vendor/bin/rspec
+
+pkg:
+	$(info => Building gem package in pkg/)
+	@mkdir pkg/
+	@gem build tfctl.gemspec
+	@mv *.gem pkg/
+
+install: pkg
+	gem install pkg/*.gem
+
+clean:
+	$(info => Cleaning)
+	@rm -rf pkg/
+	@rm -rf vendor/
+	@rm -rf .bundle
+	@rm -f Gemfile.lock
+	@rm -rf spec/reports/

data/README.adoc

@@ -0,0 +1,176 @@
+// Settings:
+:idprefix:
+:idseparator: -
+ifndef::env-github[:icons: font]
+ifdef::env-github,env-browser[]
+:toc: macro
+:toclevels: 1
+endif::[]
+ifdef::env-github[]
+:branch: master
+:status:
+:outfilesuffix: .adoc
+:!toc-title:
+:caution-caption: :fire:
+:important-caption: :exclamation:
+:note-caption: :paperclip:
+:tip-caption: :bulb:
+:warning-caption: :warning:
+endif::[]
+
+= tfctl
+
+image:https://travis-ci.org/scalefactory/tfctl.svg?branch=master["Build Status", link="https://travis-ci.org/scalefactory/tfctl"]
+image:https://badge.fury.io/rb/tfctl.svg["Gem Version", link="https://badge.fury.io/rb/tfctl"]
+image:https://img.shields.io/badge/terraform-0.12-blue.svg["Terraform 0.12", link="https://img.shields.io/badge/terraform-0.12-blue"]
+
+toc::[]
+
+== Overview
+
+Tfctl is a small Terraform wrapper for working with multi-account AWS
+infrastructures where new accounts may be created dynamically and on-demand.
+
+It discovers accounts by reading the AWS Organizations API and can assign
+Terraform resources to multiple accounts based on the organization hierarchy.
+Resources can be assigned globally, based on organization unit or to individual
+accounts.  It supports nested OU hierarchies and helps keep your Terraform DRY.
+
+Tfctl was originally developed to integrate Terraform with
+https://aws.amazon.com/solutions/aws-landing-zone/[AWS Landing Zone] and
+https://aws.amazon.com/controltower/[Control Tower] but should work with most
+other ways of managing accounts in AWS Organizations.
+
+== Features
+
+* Discovers AWS accounts automatically.
+* Automatically generates Terraform account configuration.
+* Parallel execution across multiple accounts.
+* Hierarchical configuration based on AWS Organization units structure.
+* Supports per account configuration overrides for handling exceptions.
+* Supports nested organization units.
+* Terraform state tracking in S3 and locking in DynamoDB.
+* Account targeting by OU path regular expressions.
+* Automatic role assumption in target accounts.
+* Works with CI/CD pipelines.
+
+== Requirements
+
+ * Terraform >= 0.12
+ * Ruby >= 2.3
+ * Accounts managed in AWS Organizations (by Landing Zone, Control Tower, some
+   other means)
+
+== Installation
+
+To install the latest release from RubyGems run:
+
+----
+gem install tfctl
+----
+
+Alternatively you can build and install from this repo with:
+
+----
+make install
+----
+
+== Docs
+
+ * https://github.com/scalefactory/tfctl/tree/master/docs/control_tower.adoc[Control Tower quick start guide]
+ * https://github.com/scalefactory/tfctl/tree/master/docs/project_layout.adoc[Project layout]
+ * https://github.com/scalefactory/tfctl/tree/master/docs/configuration.adoc[Configuration]
+ * https://github.com/scalefactory/tfctl/tree/master/docs/iam_permissions.adoc[IAM permissions]
+ * https://github.com/scalefactory/tfctl/tree/master/docs/creating_a_profile.adoc[Creating a profile]
+
+== Running tfctl
+
+tfctl should be run from the root of the project directory.  It will generate
+Terraform configuration in `.tfctl/`.
+
+Anatomy of a tfctl command:
+
+----
+tfctl -c CONFIG_FILE TARGET_OPTIONS -- TERRAFORM_COMMAND
+----
+
+* `-c` specifies which tfctl config file to use (usually in `conf/`)
+* `TARGET_OPTIONS` specifies which accounts to target.  This could be an individual
+  account, a group of accounts in an organizational unit or all accounts.
+* `TERRAFORM_COMMAND` will be passed to `terraform` along with any
+  options.  See https://www.terraform.io/docs/commands/index.html[Terraform
+  commands] for details.
+
+NOTE: You must have your AWS credentials configured before running tfctl or run
+it using an AWS credentials helper such as
+https://github.com/99designs/aws-vault[aws-vault].
+
+=== Example commands
+
+Show help:
+
+----
+tfctl -h
+----
+
+Show merged configuration:
+
+----
+tfctl -c conf/example.yaml -s
+----
+
+List all discovered accounts:
+
+----
+tfctl -c conf/example.yaml --all -l
+----
+
+TIP: This can be narrowed down using targeting options and is a good way to
+test what accounts match.
+
+Run Terraform init across all accounts:
+
+----
+tfctl -c conf/example.yaml --all -- init
+----
+
+Run plan in `test` OU accounts:
+
+----
+tfctl -c conf/example.yaml -o test -- plan
+----
+
+Run plan in `live` accounts assuming that `live` is a child OU in multiple
+organization units:
+
+----
+tfctl -c conf/example.yaml -o '.*/live' -- plan
+----
+
+Run plan in an individual account:
+
+----
+tfctl -c conf/example.yaml -a example-account - plan
+----
+
+Run apply in all accounts:
+
+----
+tfctl -c conf/example.yaml --all -- apply
+----
+
+Run destroy in `test` OU accounts:
+
+----
+tfctl -c conf/example.yaml -o test -- destroy -auto-approve
+----
+
+Don't buffer the output:
+
+----
+tfctl -c conf/example.yaml -a example-account -u -- plan
+----
+
+This will show output in real time.  Usually output is buffered and displayed
+after Terraform command finishes to make it more readable when running across
+multiple accounts in parallel.

data/bin/tfctl

@@ -0,0 +1,223 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+if File.directory?(File.dirname(__FILE__) + '/../vendor')
+    require 'bundler/setup'
+end
+require 'optparse'
+require 'fileutils'
+require 'parallel'
+require 'English'
+require 'terminal-table'
+require_relative '../lib/tfctl'
+
+PROJECT_ROOT = Dir.pwd
+
+#
+# Process CLI arguments
+#
+
+options = {
+    account:     nil,
+    ou:          nil,
+    all:         nil,
+    show_config: false,
+    config_file: nil,
+    unbuffered:  false,
+    debug:       false,
+    use_cache:   false,
+}
+
+optparse = OptionParser.new do |opts|
+    opts.on('-a', '--account=name', 'Target a specific AWS account') do |o|
+        options[:account] = o
+    end
+    opts.on('-o', '--ou=organization_unit', 'Target accounts in an Organization Unit (uses regex matching)') do |o|
+        options[:ou] = o
+    end
+    opts.on('--all', 'Target all accounts') do
+        options[:all] = true
+    end
+    opts.on('-c', '--config-file=config', 'Path to config file') do |o|
+        options[:config_file] = o
+    end
+    opts.on('-s', '--show-config', 'Display configuration') do
+        options[:show_config] = true
+    end
+    opts.on('-l', '--list-accounts', 'List discovered accounts') do
+        options[:list_accounts] = true
+    end
+    opts.on('-x', '--use-cache', 'Use cached AWS organization data') do
+        options[:use_cache] = true
+    end
+    opts.on('-u', '--unbuffered', 'Disable buffering of Terraform output') do
+        options[:unbuffered] = true
+    end
+    opts.on('-d', '--debug', 'Turn on debug messages') do
+        options[:debug] = true
+    end
+    opts.on('-v', '--version', 'Show version') do
+        puts Tfctl::VERSION
+        exit
+    end
+end
+
+
+begin
+    optparse.parse!
+
+    # Validate CLI arguments
+
+    if options[:config_file].nil?
+        raise OptionParser::MissingArgument, '--config-file'
+    end
+
+    unless File.exist? options[:config_file]
+        raise OptionParser::InvalidOption,
+              "Config file not found in: #{options[:config_file]}"
+    end
+
+    unless File.exist? options[:config_file]
+        raise OptionParser::InvalidOption, "Config file #{options[:config_file]} not found."
+    end
+
+    # Validate targets
+    targetting_opts = %i[account ou all]
+    targets_set = []
+    options.each do |k, v|
+        if targetting_opts.include?(k)
+            targets_set << k.to_s unless v.nil?
+        end
+    end
+    if targets_set.length > 1
+        raise OptionParser::InvalidOption,
+              "Too many target options set: #{targets_set.join(', ')}.  Only one can be specified."
+    end
+    if targets_set.empty? and options[:show_config] == false
+        raise OptionParser::InvalidOption, 'Please specify target'
+    end
+rescue OptionParser::InvalidOption, OptionParser::MissingArgument
+    warn $ERROR_INFO.to_s
+    warn optparse
+    exit 2
+end
+
+
+
+# Generates configuration and runs Terraform commands for a target account.
+def run_account(config, account, options, tf_argv, log)
+
+    # Skip excluded accounts
+    if account[:excluded] == true
+        log.info "#{account[:name]}: excluded, skipping"
+        return
+    end
+
+    # Generate Terraform run directory with configured providers, backend and
+    # profiles for the target account.  This is where Terraform will be
+    # executed from.
+    log.info "#{account[:name]}: Generating Terraform run directory"
+    Tfctl::Generator.make(
+        account: account,
+        config:  config,
+    )
+
+    log.info "#{account[:name]}: Executing Terraform #{tf_argv[0]}"
+    Tfctl::Executor.run(
+        account_name: account[:name],
+        config_name:  config[:config_name],
+        unbuffered:   options[:unbuffered],
+        log:          log,
+        argv:         tf_argv,
+    )
+end
+
+
+#
+# Main
+#
+
+begin
+    # Set up logging
+    log_level = options[:debug] ? Logger::DEBUG : Logger::INFO
+    log = Tfctl::Logger.new(log_level)
+
+    log.info 'tfctl running'
+
+    config_name = File.basename(options[:config_file]).chomp('.yaml')
+    log.info "Using config: #{config_name}"
+
+    log.info 'Working out AWS account topology'
+
+    yaml_config = YAML.safe_load(File.read(options[:config_file]))
+    Tfctl::Schema.validate(yaml_config)
+    yaml_config.symbolize_names!
+
+    org_units = yaml_config[:organization_units].keys
+    aws_org_accounts = Tfctl::AwsOrg.new(yaml_config[:tfctl_role_arn]).accounts(org_units)
+
+    log.info 'Merging configuration'
+
+    config = Tfctl::Config.new(
+        config_name:    config_name,
+        yaml_config:    yaml_config,
+        aws_org_config: aws_org_accounts,
+        use_cache:      options[:use_cache],
+    )
+
+    if options[:show_config]
+        puts config.to_yaml
+        exit 0
+    end
+
+    # Find target accounts
+
+    if options[:account]
+        accounts = config.find_accounts(:name, options[:account])
+    elsif options[:ou]
+        accounts = config.find_accounts_regex(:ou_path, options[:ou])
+    elsif options[:all]
+        accounts = config[:accounts]
+    else
+        raise Tfctl::Error, 'Missing target'
+    end
+
+    # List target accounts
+
+    if options[:list_accounts]
+        log.info "Listing accounts\n"
+        table = Terminal::Table.new do |t|
+            t.style = {
+                border_x:     '',
+                border_y:     '',
+                border_i:     '',
+                padding_left: 0,
+            }
+            t << %w[ACCOUNT_ID OU NAME]
+            accounts.each do |account|
+                t << [account[:id], account[:ou_path], account[:name]]
+            end
+        end
+
+        puts table
+        exit 0
+    end
+
+    # Execute Terraform in target accounts
+
+    Parallel.each(accounts, in_processes: 8) do |ac|
+        run_account(config, ac, options, ARGV, log)
+    end
+
+    log.info 'Done'
+rescue Tfctl::Error => e
+    log.error(e)
+    exit 1
+rescue Tfctl::ValidationError => e
+    log.error(e)
+    e.issues.each do |issue|
+        log.error("Parameter: #{issue[:data_pointer]}") unless issue[:data_pointer] == ''
+        log.error(issue[:details]) unless issue[:details].nil?
+    end
+    exit 2
+end