tfctl 1.0.0

data/docs/iam_permissions.adoc

@@ -0,0 +1,38 @@
+// Settings:
+:idprefix:
+:idseparator: -
+ifndef::env-github[:icons: font]
+ifdef::env-github,env-browser[]
+:toc: macro
+:toclevels: 1
+endif::[]
+ifdef::env-github[]
+:branch: master
+:status:
+:outfilesuffix: .adoc
+:!toc-title:
+:caution-caption: :fire:
+:important-caption: :exclamation:
+:note-caption: :paperclip:
+:tip-caption: :bulb:
+:warning-caption: :warning:
+endif::[]
+
+= IAM roles
+
+Tfctl usually requires three IAM roles to be configured:
+
+ * `TfctlRole` - read only access to AWS Organizations set up in the primary account.
+ * `TerraformStateRole` - access to remote state resources (S3, DynamoDB) in the
+    account where your state is stored (can be any account).
+ * `TerraformExecutionRole` - configured in all spoke accounts and used for executing Terraform.
+
+The user executing tfctl needs permission to assume all three roles cross
+account.  Tfctl will assume roles automatically for you.
+
+It's possible to configure different Terraform execution roles in different
+spoke accounts based on OU or account names.  This can be used to restrict
+Terraform in certain accounts.
+
+We usually set those roles up using CloudFormation as part of the bootstrapping
+process.  See example templates in `examples/bootstrap/`.

data/docs/project_layout.adoc

@@ -0,0 +1,65 @@
+// Settings:
+:idprefix:
+:idseparator: -
+ifndef::env-github[:icons: font]
+ifdef::env-github,env-browser[]
+:toc: macro
+:toclevels: 1
+endif::[]
+ifdef::env-github[]
+:branch: master
+:status:
+:outfilesuffix: .adoc
+:!toc-title:
+:caution-caption: :fire:
+:important-caption: :exclamation:
+:note-caption: :paperclip:
+:tip-caption: :bulb:
+:warning-caption: :warning:
+endif::[]
+
+= Project layout
+
+Example project structure
+----
+project_dir/
+├── conf
+│   └── example.yaml
+├── modules
+│   └── s3-bucket
+│       ├── main.tf
+│       └── variables.tf
+└── profiles
+    └── example-profile
+        ├── data.tf
+        ├── main.tf
+        └── variables.tf
+----
+
+toc::[]
+
+== tfctl configuration file
+
+Assigns Terraform profiles and configuration to accounts based on:
+
+* Globally for all accounts
+* Account's organization unit
+* Individual accounts
+
+The configuration data is exposed to terraform via a profile `config` variable.
+
+It also defines Terraform and tfctl configuration such as state tracking and
+what IAM roles to use.
+
+== profiles
+
+Profiles are re-usable collections of resources which can be applied to
+accounts.  They are implemented just like usual modules but provide an
+intermediate bridge between re-usable modules and tfctl configuration (and/or
+other data sources).  Profiles often compose multiple modules and provide
+configuration data to them.  This approach makes it possible to re-use standard
+modules (e.g. Terraform module registry).
+
+== modules
+
+Standard Terraform modules.

data/examples/bootstrap/terraform-exec-role.template

@@ -0,0 +1,24 @@
+AWSTemplateFormatVersion: 2010-09-09
+Description: Configure Terraform execution role in spoke accounts
+
+Parameters:
+  PrimaryAccountId:
+    Type: String
+
+Resources:
+  TerraformExecutionRole:
+    Type: AWS::IAM::Role
+    Properties:
+      RoleName: TerraformExecutionRole
+      AssumeRolePolicyDocument:
+        Version: 2012-10-17
+        Statement:
+          - Effect: Allow
+            Principal:
+              AWS:
+                - !Sub 'arn:aws:iam::${PrimaryAccountId}:root'
+            Action:
+              - sts:AssumeRole
+      Path: /
+      ManagedPolicyArns:
+        - arn:aws:iam::aws:policy/AdministratorAccess

data/examples/bootstrap/terraform-state.template

@@ -0,0 +1,81 @@
+AWSTemplateFormatVersion: 2010-09-09
+Description: Resources for managing Terraform state
+
+Parameters:
+
+  PrimaryAccountId:
+    Type: String
+    Description: "Primany account ID"
+  TerraformStateBucket:
+    Type: String
+    Description: "Name of S3 bucket used for storing Terraform state"
+  TerraformDynamoDbTable:
+    Type: String
+    Description: "Name of DynamoDB table used for Terraform state locking"
+    Default: "terraform-lock"
+
+Resources:
+
+  StateBucket:
+    Type: AWS::S3::Bucket
+    Properties:
+      AccessControl: Private
+      BucketName: !Ref TerraformStateBucket
+      VersioningConfiguration:
+        Status: Enabled
+      BucketEncryption:
+        ServerSideEncryptionConfiguration:
+          - ServerSideEncryptionByDefault:
+              SSEAlgorithm: AES256
+      PublicAccessBlockConfiguration:
+        BlockPublicAcls: True
+        BlockPublicPolicy: True
+        IgnorePublicAcls: True
+        RestrictPublicBuckets: True
+
+  DynamoLockTable:
+    Type: AWS::DynamoDB::Table
+    Properties:
+      TableName: !Ref TerraformDynamoDbTable
+      AttributeDefinitions:
+        - AttributeName: LockID
+          AttributeType: S
+      KeySchema:
+        - AttributeName: LockID
+          KeyType: HASH
+      ProvisionedThroughput:
+        ReadCapacityUnits: 5
+        WriteCapacityUnits: 5
+
+  TerraformStateRole:
+    Type: AWS::IAM::Role
+    Properties:
+      RoleName: TerraformStateRole
+      AssumeRolePolicyDocument:
+        Version: 2012-10-17
+        Statement:
+          - Effect: Allow
+            Principal:
+              AWS:
+                - !Sub 'arn:aws:iam::${PrimaryAccountId}:root'
+            Action:
+              - sts:AssumeRole
+      Path: /
+      Policies:
+        - PolicyName: "terraform-state"
+          PolicyDocument:
+            Version: "2012-10-17"
+            Statement:
+              - Effect: "Allow"
+                Action:
+                  - "s3:PutObject"
+                  - "s3:GetBucketPolicy"
+                  - "s3:GetObject"
+                  - "s3:ListBucket"
+                  - "dynamodb:PutItem"
+                  - "dynamodb:DeleteItem"
+                  - "dynamodb:GetItem"
+                Resource:
+                  - !GetAtt StateBucket.Arn
+                  - !Sub "${StateBucket.Arn}/*"
+                  - !GetAtt DynamoLockTable.Arn

data/examples/bootstrap/tfctl-org-access.template

@@ -0,0 +1,42 @@
+AWSTemplateFormatVersion: 2010-09-09
+Description: Create Tfctl IAM role used to workout AWS account topolgy
+
+Parameters:
+  PrimaryAccountId:
+    Type: String
+    Description: Primary organization account ID
+
+Resources:
+  TfCtlRole:
+    Type: AWS::IAM::Role
+    Properties:
+      RoleName: TfctlRole
+      AssumeRolePolicyDocument:
+        Version: '2012-10-17'
+        Statement:
+          - Effect: Allow
+            Principal:
+              AWS:
+                - !Sub 'arn:aws:iam::${PrimaryAccountId}:root'
+            Action:
+              - sts:AssumeRole
+      Path: /
+      Policies:
+        - PolicyName: TfctlOrgAccess
+          PolicyDocument:
+            Version: '2012-10-17'
+            Statement:
+              - Effect: Allow
+                Action:
+                  - organizations:ListAccounts
+                  - organizations:ListAccountsForParent
+                  - organizations:ListChildren
+                  - organizations:ListRoots
+                  - organizations:DescribeOrganizationalUnit
+                Resource:
+                  - '*'
+
+Outputs:
+  TfCtlUserArn:
+    Description: tfctl role arn
+    Value: !GetAtt 'TfCtlRole.Arn'

data/examples/control_tower/conf/example.yaml

@@ -0,0 +1,80 @@
+#
+# Example Tfctl configuration for AWS Control Tower
+#
+# The data in this file is merged with data from AWS Organizations API to
+# create final configuration used by tfctl.  You can view the merged
+# configuration by running:
+#
+#   tfctl -c conf/example.yaml -s
+#
+
+#
+# Terraform configuration
+#
+
+tf_state_bucket: 'CHANGEME'
+tf_state_dynamodb_table: 'terraform-lock'
+tf_state_region: 'eu-west-1'
+# Role for accessing state resources
+tf_state_role_arn: 'arn:aws:iam::SHARED_SERVICES_ACCOUNT_ID:role/TerraformStateRole'
+tf_required_version: '>= 0.12.0'
+aws_provider_version: '>= 2.14'
+# Role used by tfctl to retrieve data from AWS Organizations
+# Has to be set up in the primary org account
+tfctl_role_arn: 'arn:aws:iam::PRIMARY_ACCOUNT_ID:role/TfctlRole'
+
+#
+# Data
+#
+# Here you can add arbitrary data which will be accessible from Terraform
+# profiles.  Data can also be defined per account in the organization sections
+# below.
+#
+# data:
+#   my_parameter: some_value
+
+#
+# Organization configuration
+#
+# Assign resources and data to accounts based on the organization structure.
+#
+# IMPORTANT: Removing a Terraform profile here will remove all of it's
+#            associated resources during next apply!
+
+# Configuration to apply to all accounts
+organization_root:
+  # Role assumed by Terraform for execution in each account
+  tf_execution_role: 'AWSControlTowerExecution'
+  region: 'eu-west-1'
+  data:
+    # Bucket name used by example profile it will be prefixed with the target
+    # account number for uniqueness across accounts.
+    example_bucket_name: 'tfctl-example-bucket'
+  # Assign example-profile to all accounts in managed OUs
+  profiles:
+    - example-profile
+
+# Configuration to apply to accounts in Organization Units
+# OU's not listed here will be ignored.
+organization_units:
+  # Core: {} # Uncomment if you want to include Core OU accounts
+  live: {}
+  test: {}
+  mgmt:
+    data:
+      # Override the example bucket name in mgmt OU accounts
+      example_bucket_name: 'tfctl-ou-override-example'
+
+# Configuration to apply to individual accounts
+account_overrides:
+  test-example1:
+    data:
+      # Override the bucket name in a specific account
+      example_bucket_name: 'tfctl-account-override-example'
+
+
+# Exclude individual accounts from Terraform runs
+#
+# exclude_accounts:
+#    - Audit
+#    - 'Log archive'

data/examples/control_tower/modules/s3-bucket/main.tf

@@ -0,0 +1,4 @@
+resource aws_s3_bucket bucket {
+  bucket = var.name
+  acl    = "private"
+}

data/examples/control_tower/modules/s3-bucket/variables.tf

@@ -0,0 +1,4 @@
+variable "name" {
+  description = "Bucket name"
+  type = string
+}

data/examples/control_tower/profiles/example-profile/data.tf

@@ -0,0 +1 @@
+data "aws_caller_identity" "current" {}

data/examples/control_tower/profiles/example-profile/main.tf

@@ -0,0 +1,4 @@
+module "bucket" {
+  source = "../../modules/s3-bucket"
+  name   = "${local.account_id}-${local.account["example_bucket_name"]}"
+}

data/examples/control_tower/profiles/example-profile/variables.tf

@@ -0,0 +1,12 @@
+# This variable must always be present in a profile
+variable "config" {
+  description = "Configuration generated by tfctl"
+  type        = string
+}
+
+locals {
+  config     = jsondecode(var.config)
+  account_id = "${data.aws_caller_identity.current.account_id}"
+  # get current account configuration from tfctl config
+  account    = [ for account in local.config["accounts"]: account if account["id"] == local.account_id ][0]
+}

data/lib/hash.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+# Add a deep_merge method to a Hash.
+# It unions arrays (for terraform profiles behaviour)
+class Hash
+    def deep_merge(second)
+        merger = proc { |_key, v1, v2|
+            if v1.is_a?(Hash) && v2.is_a?(Hash)
+                v1.merge(v2, &merger)
+            elsif v1.is_a?(Array) && v2.is_a?(Array)
+                v1 | v2
+            elsif [:undefined, nil, :nil].include?(v2)
+                v1
+            else
+                v2
+            end
+        }
+        merge(second.to_h, &merger)
+    end
+
+    # Copied from ruby 2.6 Psych for 2.3 compatibility.
+    def symbolize_names!(result = self)
+        case result
+        when Hash
+            result.keys.each do |key|
+                result[key.to_sym] = symbolize_names!(result.delete(key))
+            end
+        when Array
+            result.map! { |r| symbolize_names!(r) }
+        end
+        result
+    end
+end

data/lib/tfctl.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require_relative 'tfctl/aws_org.rb'
+require_relative 'tfctl/config.rb'
+require_relative 'tfctl/error.rb'
+require_relative 'tfctl/executor.rb'
+require_relative 'tfctl/generator.rb'
+require_relative 'tfctl/logger.rb'
+require_relative 'tfctl/schema.rb'
+require_relative 'tfctl/version.rb'

data/lib/tfctl/aws_org.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+require_relative 'error.rb'
+require 'aws-sdk-organizations'
+
+module Tfctl
+    class AwsOrg
+
+        def initialize(role_arn)
+            @aws_org_client = Aws::Organizations::Client.new(
+                region:      'us-east-1',
+                # Assume role in primary account to read AWS organization API
+                credentials: aws_assume_role(role_arn),
+            )
+        end
+
+        # Gets account data for specified OUs from AWS Organizations API
+        def accounts(org_units)
+            output = { accounts: [] }
+
+            aws_ou_ids = aws_ou_list
+
+            org_units.each do |ou_path|
+                raise Tfctl::Error, "Error: OU: #{ou_path}, does not exists in AWS organization" unless aws_ou_ids.key?(ou_path)
+
+                parent_id = aws_ou_ids[ou_path]
+
+                @aws_org_client.list_accounts_for_parent(parent_id: parent_id).accounts.each do |account|
+                    next unless account.status == 'ACTIVE'
+
+                    output[:accounts] << {
+                        name:       account.name,
+                        id:         account.id,
+                        arn:        account.arn,
+                        email:      account.email,
+                        ou_path:    ou_path.to_s,
+                        ou_parents: ou_path.to_s.split('/'),
+                        profiles:   [],
+                    }
+                end
+            end
+            output
+        end
+
+        private
+
+        # Get a mapping of ou_name => ou_id from AWS organizations
+        def aws_ou_list
+            output = {}
+            root_ou_id = @aws_org_client.list_roots.roots[0].id
+
+            ou_recurse = lambda do |ous|
+                ous.each do |ou_name, ou_id|
+                    children = aws_ou_list_children(ou_id, ou_name)
+                    unless children.empty?
+                        output.merge!(children)
+                        ou_recurse.call(children)
+                    end
+                end
+            end
+            ou_recurse.call(root: root_ou_id)
+
+            output
+        end
+
+        # Get a list of child ou's for a parent
+        def aws_ou_list_children(parent_id, parent_name)
+            output = {}
+            retries = 0
+
+            @aws_org_client.list_children(
+                child_type: 'ORGANIZATIONAL_UNIT',
+                parent_id:  parent_id,
+            ).children.each do |child|
+
+                begin
+                    ou = @aws_org_client.describe_organizational_unit(
+                        organizational_unit_id: child.id,
+                    ).organizational_unit
+                rescue Aws::Organizations::Errors::TooManyRequestsException
+                    # FIXME: - use logger
+                    puts 'AWS Organizations: too many requests.  Retrying in 5 secs.'
+                    sleep 5
+                    retries += 1
+                    retry if retries < 10
+                end
+
+                ou_name = parent_name == :root ? ou.name.to_sym : "#{parent_name}/#{ou.name}".to_sym
+
+                output[ou_name] = ou.id
+            end
+            output
+        end
+
+        def aws_assume_role(role_arn)
+            begin
+                sts = Aws::STS::Client.new
+
+                role_credentials = Aws::AssumeRoleCredentials.new(
+                    client:            sts,
+                    role_arn:          role_arn,
+                    role_session_name: 'tfctl',
+                )
+            rescue StandardError => e
+                raise Tfctl::Error, "Error assuming role: #{role_arn}, #{e.message}"
+            end
+
+            role_credentials
+        end
+
+    end
+end