tfctl 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,169 @@
1
+ == Creating and deploying a tfctl profile
2
+
3
+ This guide will show you how to create a tfctl profile, declare some resources
4
+ in it and deploy it to to a group of accounts in an organization unit.
5
+
6
+ === Create a new profile
7
+
8
+ In your tfctl project directory create a new profile:
9
+
10
+ ----
11
+ mkdir profiles/example-profile
12
+ ----
13
+
14
+ Withing the profile create `data.tf`:
15
+
16
+ .data.tf
17
+ [source, tf]
18
+ ----
19
+ data "aws_caller_identity" "current" {}
20
+ ----
21
+
22
+ This file contains Terraform
23
+ https://www.terraform.io/docs/configuration/data-sources.html[data source]
24
+ declarations. Data sources are a way of getting data not directly managed in
25
+ Terraform into Terraform. In this case we're using the
26
+ https://www.terraform.io/docs/providers/aws/d/caller_identity.html[aws_caller_identity]
27
+ . One of the outputs of this source is `account_id` which will
28
+ return the id of the account Terraform is currently running in.
29
+
30
+ Now create `variables.tf`:
31
+
32
+ .variables.tf
33
+ [source, tf]
34
+ ----
35
+ variable "config" {
36
+ description = "Configuration generated by tfctl in string encoded JSON"
37
+ type = string
38
+ }
39
+
40
+ # local variables
41
+ locals {
42
+ # Decode config JSON into a Terraform data structure
43
+ config = jsondecode(var.config)
44
+
45
+ # Get current account id from aws_caller_identity data source
46
+ current_account_id = "${data.aws_caller_identity.current.account_id}"
47
+
48
+ # Get tfctl configuration for the current account
49
+ current_account_conf = [ for account in local.config["accounts"]: account if account["id"] == local.current_account_id ][0]
50
+ }
51
+ ----
52
+
53
+ This file contains
54
+ https://www.terraform.io/docs/configuration/variables.html[input variables] for
55
+ the profile.
56
+
57
+ The `config` variable is special and must always be declared in a tfctl
58
+ profile. Tfctl configuration can be accessed using this variable. This It
59
+ includes an array of all discovered accounts as well their parameters from
60
+ tfctl config file.
61
+
62
+ TIP: You can run `tfctl -c conf/CONFIG_FILE.yaml -s` to show the config data in
63
+ yaml format. This exact data is available in the `config` variable in your
64
+ profile.
65
+
66
+ We also have few https://www.terraform.io/docs/configuration/locals.html[local
67
+ variables] in the `locals` block. We assign the current account id from the
68
+ data source we defined previously to `current_account_id`. This is mainly for
69
+ convenience to make the next statement easier to read. `current_account` loops
70
+ over the `config` data and returns configuration for an account which matches
71
+ the current account id (i.e. the current account configuration).
72
+
73
+ Now that we have our data inputs sorted we can start declaring actual AWS
74
+ resources to manage.
75
+
76
+ Create `main.tf`:
77
+
78
+ .main.tf
79
+ [source, tf]
80
+ ----
81
+ resource "aws_s3_bucket" "example" {
82
+ bucket = "tfctl-${local.current_account_conf["name"]}"
83
+ acl = "private"
84
+ }
85
+ ----
86
+
87
+ This will create an S3 bucket with a name containing the current account name
88
+ (which will vary depending on which account it's deployed to).
89
+
90
+ === Assign profile to accounts
91
+
92
+ Before you can deploy the new profile you need to tell `tfctl` which accounts
93
+ to deploy it to.
94
+
95
+ You have few options here:
96
+
97
+ * deploy to all accounts
98
+ * deploy to specific organization unit (OU)
99
+ * deploy to individual account
100
+
101
+
102
+ For the sake of this example we're going to deploy our bucket to all accounts
103
+ in `test` OU.
104
+
105
+ In tfctl config file add the profile to the `test` OU:
106
+
107
+ [source, yaml]
108
+ ----
109
+ organization_units:
110
+ test:
111
+ profiles:
112
+ - example-profile
113
+ ----
114
+
115
+
116
+ === Plan
117
+
118
+ To see what would happen when the change is applied run:
119
+
120
+ ----
121
+ tfctl -c conf/example.yaml -o test -- init
122
+ tfctl -c conf/example.yaml -o test -- plan
123
+ ----
124
+
125
+ This will run `terraform init` to initialise terraform and then `terraform
126
+ plan` across all accounts in the `test` OU in parallel. It will display a diff
127
+ of changes for each account.
128
+
129
+ .example terraform plan
130
+ ----
131
+ info: test-example: Terraform will perform the following actions:
132
+ info: test-example:
133
+ info: test-example: # module.example-profile.aws_s3_bucket.example will be created
134
+ info: test-example: + resource "aws_s3_bucket" "example" {
135
+ info: test-example: + acceleration_status = (known after apply)
136
+ info: test-example: + acl = "private"
137
+ info: test-example: + arn = (known after apply)
138
+ info: test-example: + bucket = "tfctl-test-example"
139
+ info: test-example: + bucket_domain_name = (known after apply)
140
+ info: test-example: + bucket_regional_domain_name = (known after apply)
141
+ info: test-example: + force_destroy = false
142
+ info: test-example: + hosted_zone_id = (known after apply)
143
+ info: test-example: + id = (known after apply)
144
+ info: test-example: + region = (known after apply)
145
+ info: test-example: + request_payer = (known after apply)
146
+ info: test-example: + website_domain = (known after apply)
147
+ info: test-example: + website_endpoint = (known after apply)
148
+ info: test-example:
149
+ info: test-example: + versioning {
150
+ info: test-example: + enabled = (known after apply)
151
+ info: test-example: + mfa_delete = (known after apply)
152
+ info: test-example: }
153
+ info: test-example: }
154
+ info: test-example:
155
+ info: test-example: Plan: 1 to add, 0 to change, 0 to destroy.
156
+ ----
157
+
158
+ If there are errors in your profile, terraform will fail and usually indicate
159
+ what went wrong.
160
+
161
+ tfctl will generate a plan file automatically and use it with `apply` in the
162
+ next step.
163
+
164
+ === Apply
165
+
166
+ Once you're happy with the plan, apply it.
167
+ ----
168
+ tfctl -c conf/example.yaml -o test -- apply
169
+ ----
@@ -0,0 +1,18 @@
1
+ == IAM roles
2
+
3
+ Tfctl usually requires three IAM roles to be configured:
4
+
5
+ * `TfctlRole` - read only access to AWS Organizations set up in the primary account.
6
+ * `TerraformStateRole` - access to remote state resources (S3, DynamoDB) in the
7
+ account where your state is stored (can be any account).
8
+ * `TerraformExecutionRole` - configured in all spoke accounts and used for executing Terraform.
9
+
10
+ The user executing tfctl needs permission to assume all three roles cross
11
+ account. Tfctl will assume roles automatically for you.
12
+
13
+ It's possible to configure different Terraform execution roles in different
14
+ spoke accounts based on OU or account names. This can be used to restrict
15
+ Terraform in certain accounts.
16
+
17
+ We usually set those roles up using CloudFormation as part of the bootstrapping
18
+ process. See example templates in `examples/bootstrap/`.
@@ -0,0 +1,43 @@
1
+ == Project layout
2
+
3
+ Example project structure
4
+ ----
5
+ project_dir/
6
+ ├── conf
7
+ │   └── example.yaml
8
+ ├── modules
9
+ │   └── s3-bucket
10
+ │   ├── main.tf
11
+ │   └── variables.tf
12
+ └── profiles
13
+ └── example-profile
14
+ ├── data.tf
15
+ ├── main.tf
16
+ └── variables.tf
17
+ ----
18
+
19
+ === tfctl configuration file
20
+
21
+ Assigns Terraform profiles and configuration to accounts based on:
22
+
23
+ * Globally for all accounts
24
+ * Account's organization unit
25
+ * Individual accounts
26
+
27
+ The configuration data is exposed to terraform via a profile `config` variable.
28
+
29
+ It also defines Terraform and tfctl configuration such as state tracking and
30
+ what IAM roles to use.
31
+
32
+ === profiles
33
+
34
+ Profiles are re-usable collections of resources which can be applied to
35
+ accounts. They are implemented just like usual modules but provide an
36
+ intermediate bridge between re-usable modules and tfctl configuration (and/or
37
+ other data sources). Profiles often compose multiple modules and provide
38
+ configuration data to them. This approach makes it possible to re-use standard
39
+ modules (e.g. Terraform module registry).
40
+
41
+ === modules
42
+
43
+ Standard Terraform modules.
@@ -0,0 +1,24 @@
1
+ AWSTemplateFormatVersion: 2010-09-09
2
+ Description: Configure Terraform execution role in spoke accounts
3
+
4
+ Parameters:
5
+ PrimaryAccountId:
6
+ Type: String
7
+
8
+ Resources:
9
+ TerraformExecutionRole:
10
+ Type: AWS::IAM::Role
11
+ Properties:
12
+ RoleName: TerraformExecutionRole
13
+ AssumeRolePolicyDocument:
14
+ Version: 2012-10-17
15
+ Statement:
16
+ - Effect: Allow
17
+ Principal:
18
+ AWS:
19
+ - !Sub 'arn:aws:iam::${PrimaryAccountId}:root'
20
+ Action:
21
+ - sts:AssumeRole
22
+ Path: /
23
+ ManagedPolicyArns:
24
+ - arn:aws:iam::aws:policy/AdministratorAccess
@@ -0,0 +1,81 @@
1
+ AWSTemplateFormatVersion: 2010-09-09
2
+ Description: Resources for managing Terraform state
3
+
4
+ Parameters:
5
+
6
+ PrimaryAccountId:
7
+ Type: String
8
+ Description: "Primany account ID"
9
+ TerraformStateBucket:
10
+ Type: String
11
+ Description: "Name of S3 bucket used for storing Terraform state"
12
+ TerraformDynamoDbTable:
13
+ Type: String
14
+ Description: "Name of DynamoDB table used for Terraform state locking"
15
+ Default: "terraform-lock"
16
+
17
+ Resources:
18
+
19
+ StateBucket:
20
+ Type: AWS::S3::Bucket
21
+ Properties:
22
+ AccessControl: Private
23
+ BucketName: !Ref TerraformStateBucket
24
+ VersioningConfiguration:
25
+ Status: Enabled
26
+ BucketEncryption:
27
+ ServerSideEncryptionConfiguration:
28
+ - ServerSideEncryptionByDefault:
29
+ SSEAlgorithm: AES256
30
+ PublicAccessBlockConfiguration:
31
+ BlockPublicAcls: True
32
+ BlockPublicPolicy: True
33
+ IgnorePublicAcls: True
34
+ RestrictPublicBuckets: True
35
+
36
+ DynamoLockTable:
37
+ Type: AWS::DynamoDB::Table
38
+ Properties:
39
+ TableName: !Ref TerraformDynamoDbTable
40
+ AttributeDefinitions:
41
+ - AttributeName: LockID
42
+ AttributeType: S
43
+ KeySchema:
44
+ - AttributeName: LockID
45
+ KeyType: HASH
46
+ ProvisionedThroughput:
47
+ ReadCapacityUnits: 5
48
+ WriteCapacityUnits: 5
49
+
50
+ TerraformStateRole:
51
+ Type: AWS::IAM::Role
52
+ Properties:
53
+ RoleName: TerraformStateRole
54
+ AssumeRolePolicyDocument:
55
+ Version: 2012-10-17
56
+ Statement:
57
+ - Effect: Allow
58
+ Principal:
59
+ AWS:
60
+ - !Sub 'arn:aws:iam::${PrimaryAccountId}:root'
61
+ Action:
62
+ - sts:AssumeRole
63
+ Path: /
64
+ Policies:
65
+ - PolicyName: "terraform-state"
66
+ PolicyDocument:
67
+ Version: "2012-10-17"
68
+ Statement:
69
+ - Effect: "Allow"
70
+ Action:
71
+ - "s3:PutObject"
72
+ - "s3:GetBucketPolicy"
73
+ - "s3:GetObject"
74
+ - "s3:ListBucket"
75
+ - "dynamodb:PutItem"
76
+ - "dynamodb:DeleteItem"
77
+ - "dynamodb:GetItem"
78
+ Resource:
79
+ - !GetAtt StateBucket.Arn
80
+ - !Sub "${StateBucket.Arn}/*"
81
+ - !GetAtt DynamoLockTable.Arn
@@ -0,0 +1,42 @@
1
+ AWSTemplateFormatVersion: 2010-09-09
2
+ Description: Create Tfctl IAM role used to workout AWS account topolgy
3
+
4
+ Parameters:
5
+ PrimaryAccountId:
6
+ Type: String
7
+ Description: Primary organization account ID
8
+
9
+ Resources:
10
+ TfCtlRole:
11
+ Type: AWS::IAM::Role
12
+ Properties:
13
+ RoleName: TfctlRole
14
+ AssumeRolePolicyDocument:
15
+ Version: '2012-10-17'
16
+ Statement:
17
+ - Effect: Allow
18
+ Principal:
19
+ AWS:
20
+ - !Sub 'arn:aws:iam::${PrimaryAccountId}:root'
21
+ Action:
22
+ - sts:AssumeRole
23
+ Path: /
24
+ Policies:
25
+ - PolicyName: TfctlOrgAccess
26
+ PolicyDocument:
27
+ Version: '2012-10-17'
28
+ Statement:
29
+ - Effect: Allow
30
+ Action:
31
+ - organizations:ListAccounts
32
+ - organizations:ListAccountsForParent
33
+ - organizations:ListChildren
34
+ - organizations:ListRoots
35
+ - organizations:DescribeOrganizationalUnit
36
+ Resource:
37
+ - '*'
38
+
39
+ Outputs:
40
+ TfCtlUserArn:
41
+ Description: tfctl role arn
42
+ Value: !GetAtt 'TfCtlRole.Arn'
@@ -0,0 +1,71 @@
1
+ #
2
+ # Example Tfctl configuration for AWS Control Tower
3
+ #
4
+ # The data in this file is merged with data from AWS Organizations API to
5
+ # create final configuration used by tfctl. You can view the merged
6
+ # configuration by running:
7
+ #
8
+ # tfctl -c conf/example.yaml -s
9
+ #
10
+
11
+ #
12
+ # Terraform configuration
13
+ #
14
+
15
+ # State management
16
+
17
+ tf_state_bucket: 'CHANGEME'
18
+ tf_state_dynamodb_table: 'terraform-lock'
19
+ tf_state_region: 'eu-west-1'
20
+ # Role for accessing state resources
21
+ tf_state_role_arn: 'arn:aws:iam::SHARED_SERVICES_ACCOUNT_ID:role/TerraformStateRole'
22
+
23
+ # Role used by tfctl to retrieve data from AWS Organizations
24
+ # Has to be set up in the primary org account
25
+ tfctl_role_arn: 'arn:aws:iam::PRIMARY_ACCOUNT_ID:role/TfctlRole'
26
+
27
+ #
28
+ # Organization configuration
29
+ #
30
+
31
+ # IMPORTANT: Removing a Terraform profile here will remove all of it's
32
+ # associated resources in AWS!
33
+
34
+ # Name of the primary account.
35
+ primary_account: CHANGEME
36
+
37
+ # Configuration to apply in all accounts
38
+ organization_root:
39
+
40
+ # Role assumed by Terraform for execution in each account
41
+ tf_execution_role: 'AWSControlTowerExecution'
42
+ region: 'eu-west-1'
43
+ # Bucket name used by example profile it will be prefixed with the target
44
+ # account number for uniqueness across accounts.
45
+ example_bucket_name: 'tfctl-example-bucket'
46
+ # Assign example-profile to all accounts in managed OUs
47
+ profiles:
48
+ - example-profile
49
+
50
+ # Configuration to apply to accounts in Organization Units
51
+ # Units not listed here will be ignored
52
+ organization_units:
53
+ # Uncomment if you want to include accounts under the Core OU
54
+ # Core: {}
55
+ live: {}
56
+ test: {}
57
+ mgmt:
58
+ # Override the example bucket name in mgmt OU accounts
59
+ example_bucket_name: 'tfctl-ou-override-example'
60
+
61
+ # Configuration to apply to individual accounts
62
+ account_overrides:
63
+ test-example1:
64
+ # Override the bucket name in specific account
65
+ example_bucket_name: 'tfctl-account-override-example'
66
+
67
+
68
+ # Exclude individual accounts from Terraform runs.
69
+ #exclude_accounts:
70
+ # - Audit
71
+ # - 'Log archive'
@@ -0,0 +1,4 @@
1
+ resource aws_s3_bucket bucket {
2
+ bucket = var.name
3
+ acl = "private"
4
+ }
@@ -0,0 +1,4 @@
1
+ variable "name" {
2
+ description = "Bucket name"
3
+ type = string
4
+ }
@@ -0,0 +1 @@
1
+ data "aws_caller_identity" "current" {}
@@ -0,0 +1,4 @@
1
+ module "bucket" {
2
+ source = "../../modules/s3-bucket"
3
+ name = "${local.account_id}-${local.account["example_bucket_name"]}"
4
+ }
@@ -0,0 +1,12 @@
1
+ # This variable must always be present in a profile
2
+ variable "config" {
3
+ description = "Configuration generated by tfctl"
4
+ type = string
5
+ }
6
+
7
+ locals {
8
+ config = jsondecode(var.config)
9
+ account_id = "${data.aws_caller_identity.current.account_id}"
10
+ # get current account configuration from tfctl config
11
+ account = [ for account in local.config["accounts"]: account if account["id"] == local.account_id ][0]
12
+ }
data/lib/hash.rb ADDED
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Add a deep_merge method to a Hash.
4
+ # It unions arrays (for terraform profiles behaviour)
5
+ class ::Hash
6
+ def deep_merge(second)
7
+ merger = proc { |key, v1, v2|
8
+ Hash === v1 && Hash === v2 ? v1.merge(v2, &merger) :
9
+ Array === v1 && Array === v2 ? v1 | v2 :
10
+ [:undefined, nil, :nil].include?(v2) ? v1 : v2
11
+ }
12
+ self.merge(second.to_h, &merger)
13
+ end
14
+
15
+ # Copied from ruby 2.6 Psych for 2.3 compatibility.
16
+ def symbolize_names!(result=self)
17
+ case result
18
+ when Hash
19
+ result.keys.each do |key|
20
+ result[key.to_sym] = symbolize_names!(result.delete(key))
21
+ end
22
+ when Array
23
+ result.map! { |r| symbolize_names!(r) }
24
+ end
25
+ result
26
+ end
27
+ end
@@ -0,0 +1,121 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'error.rb'
4
+ require 'aws-sdk-organizations'
5
+
6
+ module Tfctl
7
+ class AwsOrg
8
+
9
+ def initialize(role_arn)
10
+ @aws_org_client = Aws::Organizations::Client.new(
11
+ region: 'us-east-1',
12
+ # Assume role in primary account to read AWS organization API
13
+ credentials: aws_assume_role(role_arn)
14
+ )
15
+ end
16
+
17
+ # Gets account data for specified OUs from AWS Organizations API
18
+ def accounts(org_units)
19
+ output = { :accounts => [] }
20
+
21
+ aws_ou_ids = aws_ou_list
22
+
23
+ org_units.each do |ou_path|
24
+
25
+ if aws_ou_ids.has_key?(ou_path)
26
+ parent_id = aws_ou_ids[ou_path]
27
+ else
28
+ raise Tfctl::Error.new "Error: OU: #{ou_path}, does not exists in AWS organization"
29
+ end
30
+
31
+ @aws_org_client.list_accounts_for_parent({ parent_id: parent_id }).accounts.each do |account|
32
+ if account.status == 'ACTIVE'
33
+
34
+ output[:accounts] << {
35
+ :name => account.name,
36
+ :id => account.id,
37
+ :arn => account.arn,
38
+ :email => account.email,
39
+ :ou_path => ou_path.to_s,
40
+ :ou_parents => ou_path.to_s.split('/'),
41
+ :profiles => [],
42
+ }
43
+ end
44
+ end
45
+ end
46
+ output
47
+ end
48
+
49
+ private
50
+
51
+ # Get a mapping of ou_name => ou_id from AWS organizations
52
+ def aws_ou_list()
53
+ output = {}
54
+ root_ou_id = @aws_org_client.list_roots.roots[0].id
55
+
56
+ ou_recurse = lambda do |ous|
57
+ ous.each do |ou_name, ou_id|
58
+ children = aws_ou_list_children(ou_id, ou_name)
59
+ unless children.empty?
60
+ output.merge!(children)
61
+ ou_recurse.call(children)
62
+ end
63
+ end
64
+ end
65
+ ou_recurse.call({ :root => root_ou_id })
66
+
67
+ output
68
+ end
69
+
70
+ # Get a list of child ou's for a parent
71
+ def aws_ou_list_children(parent_id, parent_name)
72
+ output = {}
73
+ retries = 0
74
+
75
+ @aws_org_client.list_children( {
76
+ child_type: 'ORGANIZATIONAL_UNIT',
77
+ parent_id: parent_id,
78
+ }).children.each do |child|
79
+
80
+ begin
81
+ ou = @aws_org_client.describe_organizational_unit({
82
+ organizational_unit_id: child.id
83
+ }).organizational_unit
84
+ rescue Aws::Organizations::Errors::TooManyRequestsException
85
+ # FIXME - use logger
86
+ puts 'AWS Organizations: too many requests. Retrying in 5 secs.'
87
+ sleep 5
88
+ retries += 1
89
+ retry if retries < 10
90
+ end
91
+
92
+ if parent_name == :root
93
+ ou_name = ou.name.to_sym
94
+ else
95
+ ou_name = "#{parent_name}/#{ou.name}".to_sym
96
+ end
97
+
98
+ output[ou_name] = ou.id
99
+ end
100
+ output
101
+ end
102
+
103
+ def aws_assume_role(role_arn)
104
+ begin
105
+ sts = Aws::STS::Client.new()
106
+
107
+ role_credentials = Aws::AssumeRoleCredentials.new(
108
+ client: sts,
109
+ role_arn: role_arn,
110
+ role_session_name: 'tfctl'
111
+ )
112
+ rescue StandardError => e
113
+ raise Tfctl::Error.new("Error assuming role: #{role_arn}, #{e.message}")
114
+ exit 1
115
+ end
116
+
117
+ role_credentials
118
+ end
119
+
120
+ end
121
+ end