tfctl 0.1.0 → 1.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cac739f38f0484a3f0a8373224183f0705c79372962c58711f82719978d86fbe
-  data.tar.gz: 8ed33267e070125bb03e1591075642163c367f8d5ab1d4ee4040c2afc97a11d8
+  metadata.gz: b7a1d287bce54f0203c80b4814f67ddb3d5f38319796a185a69ccf1fe1b65f02
+  data.tar.gz: 924e81046fde7a921f66959893429c10fbcffd8a3f32e1b6257cde2935c1b5d1
 SHA512:
-  metadata.gz: 675c2b7e868b2850749a7ca53b34ee594f84a8b30a62f3098ede1221394879d7595ecede9b76eb3da960728600092f1385ac35d122e5d2aaf17e699f903d019a
-  data.tar.gz: caf57a862a8e5ecc2e3ff64102389f1822527f4975a708407b75a23a2378d8a087fe1a6ead434f4835fc3e5681c35c145d5d5b6dfb446409ffc909940a34c2e7
+  metadata.gz: 044ad7209c428aa8c1f7eaf11258089ed6ce304a2943b79739ba1b8337fbd6487f4cda758c73acfc9de35f17fc272ef98d74fe64c274ea4d5e0112bc1749329b
+  data.tar.gz: 22a221bb105a66c5b9512a604886eb458ff1ec2876ee8d3599f7c6c0cc5e6abb5fafd0d47aa2cd61fbc979b0ba5ca33a8f18a47a5afacfd27515ea5e333f0868

data/.rubocop.yml

@@ -1,12 +1,12 @@
 ---
 AllCops:
-  TargetRubyVersion: 2.3
+  TargetRubyVersion: 2.5
   DisplayCopNames: true
 
 Layout/IndentationWidth:
   Width: 4
 
-Layout/IndentHeredoc:
+Layout/HeredocIndentation:
   Enabled: false
 
 Layout/EmptyLines:
@@ -15,7 +15,7 @@ Layout/EmptyLines:
 Layout/EmptyLinesAroundMethodBody:
   Enabled: false
 
-Layout/AlignHash:
+Layout/HashAlignment:
   EnforcedHashRocketStyle:
     - table
   EnforcedColonStyle:
@@ -45,7 +45,7 @@ Metrics/BlockLength:
 Metrics/MethodLength:
   Enabled: false
 
-Metrics/LineLength:
+Layout/LineLength:
   Max: 140
 
 Metrics/AbcSize:
@@ -77,3 +77,7 @@ Style/TrailingCommaInHashLiteral:
 
 Style/RedundantReturn:
   Enabled: false
+
+# don't break older Rubies just because of style
+Style/RedundantBegin:
+  Enabled: false

data/.travis.yml

@@ -1,17 +1,19 @@
 rvm:
-- 2.3
+- 2.5
 - 2.6
-sudo: false
+- 2.7
+os: linux
+language: ruby
 script: make test
-deploy:
-  provider: rubygems
-  api_key:
-    secure: FKAONS7x6koN7oiEULr4ViwjlDBzbE0bCgqhXRP4DfTtlRyEymeqgrfSIqkVH7unjd0muIGrMBnFuaQVSx7648RS1Ss0QAJo32SVnzoYl1P03cijqNmbbaf1jRdA3IGmh0gV5vsXrmlHiP8gfAuC9PqQ550OzxzWUvEI8vXgSTibmKd/PoQinv5g/dq0gBFjlhSMt/k3Z9WlMmkEsAro/r/Ie2M7mItHPT65f0ga5q5SeujPQQ3Sd/l3mznh37bmnw5RZpFDYdA7jL2p0Y58XJPBU8soa3ZC5GeHyxCYVoGh6EDGAFb83ERRT6rQ7ywkOufTv1o497P7a/prSbvT6fzc+DcugXPEaglT+dUXMe36OoF907Xva4vq3xIHV2N/yrxbDM85hmMk22wEU+9wpDDzFNQnfsXNbaHG9F7gLgy0eoTRrSuJf6cPDlE8pwvn7b8cjieeqWc//ZNhSYnHYZGER4LFINWVxs68Eofmmqp2IESTcUpJ8oB4bV+bzzyobJMRobOXu2hvgCrTdr6r/PnckpAfZE/l4nVQa14f1FU//8bU3DwvNun6TX1Ujp+XNiRDUlvP2KnkBU4s5rsIkL3lCHW7r6GipSk6SOvGMTz5eySMsoWvZQBdAzk/OxcIteeWH9pdo1Hbu5x2/bwyuTRCQ9E79CKWDKlIQwCgUY0=
-  gem: tfctl
-  on:
-    tags: true
-    repo: scalefactory/tfctl
-notifications:
-  slack:
-    rooms:
-      secure: HrdsUaTCvwtq5rLyAf7Uwo76p9JhxnoFvoZU/AtxTCL6lwMm5pqpfDAdbBH8C2BYMWRyiS8IfSrvuLbGUr317qJgVCmnhD7JIgdA0Aih4GkR0yK+EVao7qdOClO2CGJFJGqHveBBpXy94FEwgFwQSvnuSZdY8gNdXafvpgrPvcbWHIF9Bq7StfYelWZX6+FL7xczqVhdetXcHJEj6Q/thkbvyn0hJAVegjDq4Pw3MyR66Mv5HTj+jYbGrJOR1poNW8vv4iQjK0TIMvCCeTv4Ddrjitn1kOK1BPYqoHEkaRwXdS7+hevv+UfV074awTp3UOIS5VP3ve/KX21xcl9DWMG7Gde9tuyap9WKhN3XE5686qrfn4M/L7PmVnzeYimUKLQfdqjDXa7UI3orGIOq8nKCsCUFvFUQZ/DxeEssJzBNNRbeGywwVtXk7/8L2HHiQJ3t/29vMLEq6EwskYK3Uao2aIVWpLSkPwOuwVxrNWXrwojrrKo+oo8QvW6xkov5UW3VkapyzRyDN2oIpNkFeLYfuyTO4MBsnLzCBqJYEeku2l0Hp9WBq2Q1a+FRgqQlcRbLPsYwK5AdkRKcWuXP8FMZoqXX6F9B73q5Dbub2et4YN4zhvroLZN6MmlneM6faJh9aS4yV2ZlU1XbOtbqP0nkHrJ+o8DeM/wgTB1GB90=
+jobs:
+  include:
+  - stage: Gem release
+    rvm: 2.6
+    deploy:
+      provider: rubygems
+      api_key:
+        secure: LAVcdER+LtQ2TSUrVOY7Be1BC7GXJRD0QBt386vRM5Nld5QaD9Ow9gtN6FprzkzloI4R8BkPWqZbAT6YjC+C0AFB5HK6iPwD2bLsiF9w3ccDD+yrW99RHxiErpmYMun2PqZv0WkJ/pkEplPCKMRFv7SM7W9DMRlU7dsXc1v6IVyIb5u3A04jErS2jXXKY0ijlCDJYVo8zzYL6yUmUcXhc//3CIVnu2Miu6Qr8h7e6jMXNUWfMkwEXsFP9id4TsCz7hRY+39PkiBAknHTN5UqjjJiEOknZnHeTBcVPvi2h2xv+fFLSzVTxlxaRsVoMCShQp5D12qzhQObRJsRVQFs8Yyg9IYMyPdxssFYyUZFaAy5taWDm57uM3HTHylm/Dq3LmXTgGNxWUUkf2oh1g7R6cYZpBUQwiEPzhZQ7CoBQbGUAJmH9ZU9m+cr8kuAOUipd6BNEDvn/fIH4WJsRCNP72JGX16JBpuICvpkuNhskZT91xFlYk1pTXHOxNpbTcxUTgMHhrTqspeRXPmf6DiYGvjMb2S6kaoGqCIRIcwl0TGKuMsOMqR9SqF8gubkqHMVbSl1E7mwBn4ke8/7IGoMkWOGwUpVxqVOLBHi6zSR09RTVSbKl4oiFV3ZwmVPSxDncq54MptyJ2WCZ7dD6ht2l+VA8iGwYeIoqOpwGWxyuNI=
+      gem: tfctl
+      on:
+        tags: true
+        repo: scalefactory/tfctl

data/CHANGELOG.adoc

@@ -1,12 +1,40 @@
 = Changelog
 
+== Upcoming
+
+== 1.1.1
+
+ * fix: handle empty response from Organizations API containing children (thanks @grothja)
+ * chore: stopped testing on EOL Rubies 2.3 and 2.4 (but should still currently work)
+ * chore: dependencies minimum version bump
+
+== 1.1.0
+
+* feat: look for configuration in `tfctl.yaml` by default.
+
+== 1.0.0
+
+* feat(config): JSON schema config validation
+* feat(config): added 'data' parameter
+
+BREAKING CHANGE: This release moves user defined data under a separate `data`
+parameter so it can be easily distinguished from parameters required by tfctl.
+Configuration file will need to be updated to reflect this to pass validation.
+
+
+== 0.2.0
+
+* feat: configurable Terraform and AWS provider version requirements
+* fix: use provider region from config file
+* fix: fail when terraform command is missing
+
 == 0.1.0
 
-* FEATURE: Added `-l` switch to list discovered accounts.
+* feat: Added `-l` switch to list discovered accounts.
 
 == 0.0.2
 
-* BUGFIX: Fixed an exception when `exclude_accounts` is not set.
+* fix: Fixed an exception when `exclude_accounts` is not set.
 
 == 0.0.1
 

data/Guardfile

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+guard :rspec, cmd: 'bundle exec rspec' do
+    watch(%r{^spec/.+_spec\.rb$})
+    watch(%r{^lib/tfctl/(.+)\.rb$}) { |m| "spec/#{m[1]}_spec.rb" }
+    watch('spec/spec_helper.rb')  { 'spec' }
+end

data/Makefile

@@ -1,4 +1,4 @@
-.PHONY: clean install test rubocop spec
+.PHONY: clean install test rubocop spec guard
 
 vendor:
 	$(info => Installing Ruby dependencies)
@@ -6,6 +6,10 @@ vendor:
 
 test: vendor rubocop spec
 
+guard: vendor
+	$(info => Starting guard)
+	@bundle exec guard
+
 rubocop:
 	$(info => Running rubocop)
 	@vendor/bin/rubocop

data/README.adoc

@@ -1,21 +1,43 @@
-:toc:
+// Settings:
+:idprefix:
+:idseparator: -
+ifndef::env-github[:icons: font]
+ifdef::env-github,env-browser[]
+:toc: macro
+:toclevels: 1
+endif::[]
+ifdef::env-github[]
+:branch: master
+:status:
+:outfilesuffix: .adoc
+:!toc-title:
+:caution-caption: :fire:
+:important-caption: :exclamation:
+:note-caption: :paperclip:
+:tip-caption: :bulb:
+:warning-caption: :warning:
+endif::[]
 
 = tfctl
 
 image:https://travis-ci.org/scalefactory/tfctl.svg?branch=master["Build Status", link="https://travis-ci.org/scalefactory/tfctl"]
 image:https://badge.fury.io/rb/tfctl.svg["Gem Version", link="https://badge.fury.io/rb/tfctl"]
+image:https://img.shields.io/badge/terraform-0.12-blue.svg["Terraform 0.12", link="https://img.shields.io/badge/terraform-0.12-blue"]
+
+toc::[]
 
 == Overview
 
-Tfctl is a small Terraform wrapper for working with multi-account AWS
+`tfctl` is a small Terraform wrapper for working with multi-account AWS
 infrastructures where new accounts may be created dynamically and on-demand.
 
-Discovers accounts by reading the AWS Organizations API and can assign
-Terraform resources to accounts based on the organization hierarchy.  Resources
-can be assigned globally, based on organization unit or individual accounts.
-It supports nested OU hierarchies.
+It discovers accounts by reading the AWS Organizations API, and can assign
+Terraform resources to multiple accounts based on the organization hierarchy.
+Resources can be assigned globally, based on organization unit or to individual
+accounts.  It supports hierarchies of nested Organizational Units (OUs),
+and helps keep your Terraform DRY.
 
-Tfctl was originally developed to integrate Terraform with
+The Scale Factory originally created tfctl to integrate Terraform with
 https://aws.amazon.com/solutions/aws-landing-zone/[AWS Landing Zone] and
 https://aws.amazon.com/controltower/[Control Tower] but should work with most
 other ways of managing accounts in AWS Organizations.
@@ -36,7 +58,7 @@ other ways of managing accounts in AWS Organizations.
 == Requirements
 
  * Terraform >= 0.12
- * Ruby >= 2.3
+ * Ruby >= 2.4
  * Accounts managed in AWS Organizations (by Landing Zone, Control Tower, some
    other means)
 
@@ -44,17 +66,19 @@ other ways of managing accounts in AWS Organizations.
 
 To install the latest release from RubyGems run:
 
+[source,shell]
 ----
 gem install tfctl
 ----
 
-Alternatively you can build and install from this repo with:
+Alternatively, you can build and install from this repo with:
 
+[source,shell]
 ----
 make install
 ----
 
-== Docs
+== Documentation
 
  * https://github.com/scalefactory/tfctl/tree/master/docs/control_tower.adoc[Control Tower quick start guide]
  * https://github.com/scalefactory/tfctl/tree/master/docs/project_layout.adoc[Project layout]
@@ -64,23 +88,25 @@ make install
 
 == Running tfctl
 
-tfctl should be run from the root of the project directory.  It will generate
-Terraform configuration in `.tfctl/`.
+You should run `tfctl` from the root of your project directory.  It will generate
+Terraform configuration in `.tfctl/` (add this to your `.gitignore`).
 
 Anatomy of a tfctl command:
 
+[source,shell]
 ----
 tfctl -c CONFIG_FILE TARGET_OPTIONS -- TERRAFORM_COMMAND
 ----
 
-* `-c` specifies which tfctl config file to use (usually in `conf/`)
+* `-c` specifies which tfctl config file to use (defaults to `tfctl.yaml` in
+ current working directory if not set)
 * `TARGET_OPTIONS` specifies which accounts to target.  This could be an individual
   account, a group of accounts in an organizational unit or all accounts.
 * `TERRAFORM_COMMAND` will be passed to `terraform` along with any
   options.  See https://www.terraform.io/docs/commands/index.html[Terraform
   commands] for details.
 
-NOTE: You must have your AWS credentials configured before running tfctl or run
+NOTE: You must have your AWS credentials configured before you run `tfctl`, or run
 it using an AWS credentials helper such as
 https://github.com/99designs/aws-vault[aws-vault].
 
@@ -88,68 +114,78 @@ https://github.com/99designs/aws-vault[aws-vault].
 
 Show help:
 
+[source,shell]
 ----
 tfctl -h
 ----
 
 Show merged configuration:
 
+[source,shell]
 ----
-tfctl -c conf/example.yaml -s
+tfctl -s
 ----
 
 List all discovered accounts:
 
+[source,shell]
 ----
-tfctl -c conf/example.yaml --all -l
+tfctl --all -l
 ----
 
 TIP: This can be narrowed down using targeting options and is a good way to
 test what accounts match.
 
-Run Terraform init across all accounts:
+Run `terraform init` across all accounts:
 
+[source,shell]
 ----
-tfctl -c conf/example.yaml --all -- init
+tfctl --all -- init
 ----
 
-Run plan in `test` OU accounts:
+Plan Terraform across all accounts in the `test` OU:
 
+[source,shell]
 ----
-tfctl -c conf/example.yaml -o test -- plan
+tfctl -o test -- plan
 ----
 
-Run plan in `live` accounts assuming that `live` is a child OU in multiple
+Plan Terraform in `live` accounts, assuming that `live` is a child OU in multiple
 organization units:
 
+[source,shell]
 ----
-tfctl -c conf/example.yaml -o '.*/live' -- plan
+tfctl -o '.*/live' -- plan
 ----
 
-Run plan in an individual account:
+Run a plan for an individual account:
 
+[source,shell]
 ----
-tfctl -c conf/example.yaml -a example-account - plan
+tfctl -a example-account - plan
 ----
 
-Run apply in all accounts:
+Apply Terraform changes across all accounts:
 
+[source,shell]
 ----
-tfctl -c conf/example.yaml --all -- apply
+tfctl --all -- apply
 ----
 
-Run destroy in `test` OU accounts:
+Destroy Terraform-managed resources in all the `test` OU accounts:
 
+[source,shell]
 ----
-tfctl -c conf/example.yaml -o test -- destroy -auto-approve
+tfctl -o test -- destroy -auto-approve
 ----
 
 Don't buffer the output:
 
+[source,shell]
 ----
-tfctl -c conf/example.yaml -a example-account -u -- plan
+tfctl -a example-account -u -- plan
 ----
 
 This will show output in real time.  Usually output is buffered and displayed
-after Terraform command finishes to make it more readable when running across
-multiple accounts in parallel.
+after the Terraform command finishes, to make it more readable when running
+across multiple accounts in parallel.

data/bin/tfctl

@@ -22,7 +22,7 @@ options = {
     ou:          nil,
     all:         nil,
     show_config: false,
-    config_file: nil,
+    config_file: 'tfctl.yaml',
     unbuffered:  false,
     debug:       false,
     use_cache:   false,
@@ -68,10 +68,6 @@ begin
 
     # Validate CLI arguments
 
-    if options[:config_file].nil?
-        raise OptionParser::MissingArgument, '--config-file'
-    end
-
     unless File.exist? options[:config_file]
         raise OptionParser::InvalidOption,
               "Config file not found in: #{options[:config_file]}"
@@ -104,7 +100,7 @@ end
 
 
 
-# Generates configuration and runs Terraform commands for a target account.
+# Execute terraform in target accounts
 def run_account(config, account, options, tf_argv, log)
 
     # Skip excluded accounts
@@ -118,11 +114,8 @@ def run_account(config, account, options, tf_argv, log)
     # executed from.
     log.info "#{account[:name]}: Generating Terraform run directory"
     Tfctl::Generator.make(
-        config:         config,
-        account_id:     account[:id],
-        account_name:   account[:name],
-        profiles:       account[:profiles],
-        execution_role: account[:tf_execution_role],
+        account: account,
+        config:  config,
     )
 
     log.info "#{account[:name]}: Executing Terraform #{tf_argv[0]}"
@@ -148,11 +141,13 @@ begin
     log.info 'tfctl running'
 
     config_name = File.basename(options[:config_file]).chomp('.yaml')
+    config_name = 'default' if config_name == 'tfctl'
     log.info "Using config: #{config_name}"
 
     log.info 'Working out AWS account topology'
 
     yaml_config = YAML.safe_load(File.read(options[:config_file]))
+    Tfctl::Schema.validate(yaml_config)
     yaml_config.symbolize_names!
 
     org_units = yaml_config[:organization_units].keys
@@ -215,4 +210,11 @@ begin
 rescue Tfctl::Error => e
     log.error(e)
     exit 1
+rescue Tfctl::ValidationError => e
+    log.error(e)
+    e.issues.each do |issue|
+        log.error("Parameter: #{issue[:data_pointer]}") unless issue[:data_pointer] == ''
+        log.error(issue[:details]) unless issue[:details].nil?
+    end
+    exit 2
 end

data/docs/configuration.adoc

@@ -1,7 +1,31 @@
-== Configuration
+// Settings:
+:idprefix:
+:idseparator: -
+ifndef::env-github[:icons: font]
+ifdef::env-github,env-browser[]
+:toc: macro
+:toclevels: 1
+endif::[]
+ifdef::env-github[]
+:branch: master
+:status:
+:outfilesuffix: .adoc
+:!toc-title:
+:caution-caption: :fire:
+:important-caption: :exclamation:
+:note-caption: :paperclip:
+:tip-caption: :bulb:
+:warning-caption: :warning:
+endif::[]
 
-Tfctl retrieves initial account configuration from AWS Organizations and merges
-it with organization config specified in the yaml file.
+= Configuration
+
+toc::[]
+
+== Overview
+
+`tfctl` retrieves initial account configuration from AWS Organizations and merges
+it with configuration specified in YAML format (`tfctl.yaml` by default).
 
 The configuration is merged in the following order:
 
@@ -15,13 +39,16 @@ Parameters further down the hierarchy take precedence.  For example:
 [source, yaml]
 ----
 organization_root:
-  example_param: 'will be overriden further down'
+  data:
+    example_param: 'will be overriden further down'
 
 organization_units:
   team:
-    example_param: 'will win in team ou'
+    data:
+      example_param: 'will win in team ou'
   team/live:
-    example_param: 'will win in team/live ou'
+    data:
+      example_param: 'will win in team/live ou'
 ----
 
 One exception to this rule is the `profiles` parameter.  Profiles are additive:
@@ -41,13 +68,32 @@ organization_units:
 
 This will result in all three profiles deployed to accounts in `team` OU.
 
-TIP: You can display the fully merged configuration by running `tfctl -c
-conf/CONFIG_FILE.yaml -s`.  It's safe to run as it doesn't make any changes to
-AWS resources.  It's a good way to test your configuration.
+TIP: You can display the fully merged configuration by running `tfctl -s`.
+It's safe to run as it doesn't make any changes to AWS resources.  It's a good
+way to test your configuration.
+
+== Defining arbitrary data
+
+You can define arbitrary data under the `data:` parameter, both in the root of
+the config and in the organization sections.  It will be available in Terraform
+profiles to use by your modules.  You can use this to define things like VPC
+subnet ranges, s3 bucket names and so on.  `data:` in organization sections
+will be merged with accounts following the usual merge order as described
+above.
 
-=== Handling secrets
+== Handling secrets
 
-No secrets should be committed into Terraform or tfctl configuration.  Use AWS
-Secrets Manager instead and retrieve in Terraform profiles using
+CAUTION: Do not commit secrets into your Terraform or tfctl configuration.
+
+Instead, use AWS Secrets Manager and retrieve secrets in Terraform profiles using
+the
 https://www.terraform.io/docs/providers/aws/d/secretsmanager_secret.html[secrets
-manager data source]
+manager data source].
+
+== Configuration Schema
+
+The configuration file is validated using https://json-schema.org/[JSON Schema].
+
+The schema is defined in
+https://github.com/scalefactory/tfctl/blob/master/lib/tfctl/schema.rb[lib/tfctl/schema.rb]
+and is a good place to look up all available options.