tfctl 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,177 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative '../hash.rb'
4
+ require_relative 'error.rb'
5
+ require 'yaml'
6
+ require 'json'
7
+
8
+ module Tfctl
9
+ class Config
10
+ include Enumerable
11
+ attr_reader :config
12
+
13
+ def initialize(config_name:, yaml_config:, aws_org_config:, use_cache: false)
14
+ cache_file = "#{PROJECT_ROOT}/.tfctl/#{config_name}_cache.yaml"
15
+
16
+ # Get configuration. Either load from cache or process fresh.
17
+ if use_cache
18
+ @config = read_cache(cache_file)
19
+ else
20
+ @config = load_config(config_name, yaml_config, aws_org_config)
21
+ write_cache(@config, cache_file)
22
+ end
23
+ end
24
+
25
+ def [](key)
26
+ @config[key]
27
+ end
28
+
29
+ def each(&block)
30
+ @config.each(&block)
31
+ end
32
+
33
+ def has_key?(k)
34
+ @config.has_key?(k)
35
+ end
36
+
37
+ def to_yaml
38
+ @config.to_yaml
39
+ end
40
+
41
+ def to_json
42
+ @config.to_json
43
+ end
44
+
45
+ # Filters accounts by account property
46
+ def find_accounts(property_name, property_value)
47
+ output =[]
48
+ @config[:accounts].each do |account|
49
+ if account[property_name] == property_value
50
+ output << account
51
+ end
52
+ end
53
+
54
+ if output.empty?
55
+ raise Tfctl::Error.new "Account not found with #{property_name}: #{property_value}"
56
+ end
57
+ output
58
+ end
59
+
60
+ def find_accounts_regex(property_name, expr)
61
+ output =[]
62
+ @config[:accounts].each do |account|
63
+ begin
64
+ if account[property_name] =~ /#{expr}/
65
+ output << account
66
+ end
67
+ rescue RegexpError => e
68
+ raise Tfctl::Error.new "Regexp: #{e}"
69
+ end
70
+ end
71
+
72
+ if output.empty?
73
+ raise Tfctl::Error.new "Account not found with #{property_name} matching regex: #{expr}"
74
+ end
75
+ output
76
+ end
77
+
78
+
79
+ private
80
+
81
+ # Retrieves AWS Organizations data and merges it with data from yaml config.
82
+ def load_config(config_name, yaml_config, aws_org_config)
83
+
84
+ # AWS Organizations data
85
+ config = aws_org_config
86
+ # Merge organization sections from yaml file
87
+ config = merge_accounts_config(config, yaml_config)
88
+ # Import remaining parameters from yaml file
89
+ config = import_yaml_config(config, yaml_config)
90
+ # Set excluded property on any excluded accounts
91
+ config = mark_excluded_accounts(config)
92
+ # Remove any profiles that are unset
93
+ config = remove_unset_profiles(config)
94
+ # Set config name property (based on yaml config file name)
95
+ config[:config_name] = config_name
96
+ config
97
+ end
98
+
99
+ def write_cache(config, cache_file)
100
+ FileUtils.mkdir_p File.dirname(cache_file)
101
+ File.open(cache_file, 'w') {|f| f.write self.to_yaml }
102
+ end
103
+
104
+ def read_cache(cache_file)
105
+ unless File.exist?(cache_file)
106
+ raise Tfctl::Error.new("Cached configuration not found in: #{cache_file}")
107
+ end
108
+
109
+ YAML.load_file(cache_file)
110
+ end
111
+
112
+ # Sets :excluded property on any excluded accounts
113
+ def mark_excluded_accounts(config)
114
+ return config unless config.has_key?(:exclude_accounts)
115
+
116
+ config[:accounts].each_with_index do |account, idx|
117
+ if config[:exclude_accounts].include?(account[:name])
118
+ config[:accounts][idx][:excluded] = true
119
+ else
120
+ config[:accounts][idx][:excluded] = false
121
+ end
122
+ end
123
+ config
124
+ end
125
+
126
+ def remove_unset_profiles(config)
127
+ config[:accounts].each do |account|
128
+ profiles_to_unset = []
129
+ account[:profiles].each do |profile|
130
+ if profile =~ /\.unset$/
131
+ profiles_to_unset << profile
132
+ profiles_to_unset << profile.chomp('.unset')
133
+ end
134
+ end
135
+ account[:profiles] = account[:profiles] - profiles_to_unset
136
+ end
137
+ config
138
+ end
139
+
140
+ # Import yaml config other than organisation defaults sections which are merged elsewhere.
141
+ def import_yaml_config(config, yaml_config)
142
+ yaml_config.delete(:organization_root)
143
+ yaml_config.delete(:organization_units)
144
+ yaml_config.delete(:account_overrides)
145
+ config.merge(yaml_config)
146
+ end
147
+
148
+ # Merge AWS Organizations accounts config with defaults from yaml config
149
+ def merge_accounts_config(config, yaml_config)
150
+
151
+ config[:accounts].each_with_index do |account_config, idx|
152
+ account_name = account_config[:name].to_sym
153
+ account_ou_parents = account_config[:ou_parents]
154
+
155
+ # merge any root settings
156
+ account_config = account_config.deep_merge(yaml_config[:organization_root])
157
+
158
+ # merge all OU levels settings
159
+ account_ou_parents.each_with_index do |_, i|
160
+ account_ou = account_ou_parents[0..i].join('/').to_sym
161
+ if yaml_config[:organization_units].has_key?(account_ou)
162
+ account_config = account_config.deep_merge(yaml_config[:organization_units][account_ou])
163
+ end
164
+ end
165
+
166
+ # merge any account overrides
167
+ if yaml_config[:account_overrides].has_key?(account_name)
168
+ account_config = account_config.deep_merge(yaml_config[:account_overrides][account_name])
169
+ end
170
+
171
+ config[:accounts][idx] = account_config
172
+ end
173
+ config
174
+ end
175
+
176
+ end
177
+ end
@@ -0,0 +1,6 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Tfctl
4
+ class Error < StandardError
5
+ end
6
+ end
@@ -0,0 +1,104 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'open3'
4
+ require 'fileutils'
5
+ require 'shellwords'
6
+ require 'thread'
7
+ require_relative 'error.rb'
8
+
9
+ module Tfctl
10
+ module Executor
11
+ extend self
12
+
13
+ # Execute terraform command
14
+ def run(account_name:, config_name:, log:, cmd: nil, argv: [], unbuffered: true)
15
+
16
+ if cmd.nil?
17
+ if File.exists?("#{PROJECT_ROOT}/bin/terraform")
18
+ # use embedded terraform binary
19
+ cmd = "#{PROJECT_ROOT}/bin/terraform"
20
+ else
21
+ cmd = 'terraform'
22
+ end
23
+ end
24
+
25
+ path = "#{PROJECT_ROOT}/.tfctl/#{config_name}/#{account_name}"
26
+ cwd = FileUtils.pwd
27
+ plan_file = "#{path}/tfplan"
28
+ semaphore = Mutex.new
29
+ output = []
30
+
31
+ # Extract terraform sub command from argument list
32
+ args = Array.new(argv)
33
+ subcmd = args[0]
34
+ args.delete_at(0)
35
+
36
+ # Enable plan file for `plan` and `apply` sub commands
37
+ args += plan_file_args(plan_file, subcmd)
38
+
39
+ # Create the command
40
+ exec = [cmd] + [subcmd] + args
41
+
42
+ # Set environment variables for Terraform
43
+ env = {
44
+ 'TF_INPUT' => '0',
45
+ 'CHECKPOINT_DISABLE' => '1',
46
+ 'TF_IN_AUTOMATION' => 'true',
47
+ # 'TF_LOG' => 'TRACE'
48
+ }
49
+
50
+ log.debug "#{account_name}: Executing: #{exec.shelljoin}"
51
+
52
+ FileUtils.cd path
53
+ Open3.popen3(env, exec.shelljoin) do |stdin, stdout, stderr, wait_thr|
54
+ stdin.close_write
55
+
56
+ # capture stdout and stderr in separate threads to prevent deadlocks
57
+ Thread.new do
58
+ stdout.each do |line|
59
+ semaphore.synchronize do
60
+ unbuffered ? log.info("#{account_name}: #{line.chomp}") : output << [ 'info', line ]
61
+ end
62
+ end
63
+ end
64
+ Thread.new do
65
+ stderr.each do |line|
66
+ semaphore.synchronize do
67
+ unbuffered ? log.error("#{account_name}: #{line.chomp}") : output << [ 'error', line ]
68
+ end
69
+ end
70
+ end
71
+
72
+ status = wait_thr.value
73
+
74
+ # log the output
75
+ output.each do |line|
76
+ log.send(line[0], "#{account_name}: #{line[1].chomp}")
77
+ end
78
+
79
+ FileUtils.cd cwd
80
+ FileUtils.rm_f plan_file if args[0] == 'apply' # tidy up the plan file
81
+
82
+ unless status.exitstatus == 0
83
+ raise Tfctl::Error.new "#{cmd} failed with exit code: #{status.exitstatus}"
84
+ end
85
+ end
86
+ end
87
+
88
+ # Adds plan file to `plan` and `apply` sub commands
89
+ def plan_file_args(plan_file, subcmd)
90
+ output = []
91
+ if subcmd == 'plan'
92
+ output = [ "-out=#{plan_file}" ]
93
+
94
+ elsif subcmd == 'apply'
95
+ if File.exists?(plan_file)
96
+ output = [ "#{plan_file}" ]
97
+ else
98
+ raise Tfctl::Error.new "Plan file not found in #{plan_file}. Run plan first."
99
+ end
100
+ end
101
+ output
102
+ end
103
+ end
104
+ end
@@ -0,0 +1,96 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'fileutils'
4
+
5
+ # Generates Terraform configuration for an account.
6
+
7
+ module Tfctl
8
+ module Generator
9
+ extend self
10
+
11
+ def write_json_block(path, block)
12
+ File.open(path, 'w') do |f|
13
+ f.write(JSON.pretty_generate(block) + "\n")
14
+ end
15
+ end
16
+
17
+ def make(
18
+ terraform_io_org:,
19
+ account_id:,
20
+ account_name:,
21
+ execution_role:,
22
+ profiles:,
23
+ config:,
24
+ region: 'eu-west-1',
25
+ tf_version: '>= 0.12.0',
26
+ aws_provider_version: '~> 2.14',
27
+ target_dir: "#{PROJECT_ROOT}/.tfctl/#{config[:config_name]}/#{account_name}"
28
+ )
29
+
30
+ FileUtils.mkdir_p target_dir
31
+
32
+ terraform_block = {
33
+ 'terraform' => {
34
+ 'required_version' => tf_version,
35
+ 'backend' => {
36
+ 's3' => {
37
+ 'bucket' => config[:tf_state_bucket],
38
+ 'key' => "#{account_name}/tfstate",
39
+ 'region' => config[:tf_state_region],
40
+ 'role_arn' => config[:tf_state_role_arn],
41
+ 'dynamodb_table' => config[:tf_state_dynamodb_table],
42
+ 'encrypt' => 'true',
43
+ }
44
+ }
45
+ }
46
+ }
47
+ write_json_block("#{target_dir}/terraform.tf.json", terraform_block)
48
+
49
+ provider_block = {
50
+ 'provider' => {
51
+ 'aws' => {
52
+ 'version' => aws_provider_version,
53
+ 'region' => region,
54
+ 'assume_role' => {
55
+ 'role_arn' => "arn:aws:iam::#{account_id}:role/#{execution_role}"
56
+ }
57
+ }
58
+ }
59
+ }
60
+ write_json_block("#{target_dir}/provider.tf.json", provider_block)
61
+
62
+ vars_block = {
63
+ 'variable' => {
64
+ 'config' => {
65
+ 'type' => 'string'
66
+ }
67
+ }
68
+ }
69
+ write_json_block("#{target_dir}/vars.tf.json", vars_block)
70
+
71
+ # config is passed to profiles as a json encoded string. It can be
72
+ # decoded in profile using jsondecode() function.
73
+ config_block = { 'config' => config.to_json }
74
+ write_json_block("#{target_dir}/config.auto.tfvars.json", config_block)
75
+
76
+ FileUtils.rm Dir.glob("#{target_dir}/profile_*.tf.json")
77
+
78
+ profiles.each do |profile|
79
+ profile_block = {
80
+ 'module' => {
81
+ profile => {
82
+ 'source' => "../../../profiles/#{profile}",
83
+ 'config' => '${var.config}',
84
+ 'providers' => {
85
+ 'aws' => 'aws'
86
+ }
87
+ }
88
+ }
89
+ }
90
+
91
+ write_json_block("#{target_dir}/profile_#{profile}.tf.json", profile_block)
92
+ end
93
+
94
+ end
95
+ end
96
+ end
@@ -0,0 +1,38 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'logger'
4
+
5
+ module Tfctl
6
+ class Logger
7
+
8
+ def initialize(log_level)
9
+ @outlog = ::Logger.new(STDOUT)
10
+
11
+ self.level = log_level
12
+
13
+ @outlog.formatter = proc do |severity, datetime, progname, msg|
14
+ # "#{datetime.iso8601} #{severity.downcase}: #{msg}\n"
15
+ "#{severity.downcase}: #{msg}\n"
16
+ end
17
+ end
18
+
19
+ def level=(level)
20
+ @outlog.level = level
21
+ end
22
+
23
+ def level
24
+ @outlog.level
25
+ end
26
+
27
+ def debug(msg); log(:debug, msg); end
28
+ def info(msg); log(:info, msg); end
29
+ def warn(msg); log(:warn, msg); end
30
+ def error(msg); log(:error, msg); end
31
+ def fatal(msg); log(:fatal, msg); end
32
+
33
+ def log(level, msg)
34
+ @outlog.send(level, msg)
35
+ end
36
+
37
+ end
38
+ end
@@ -0,0 +1,5 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Tfctl
4
+ VERSION = '0.0.2'
5
+ end
data/lib/tfctl.rb ADDED
@@ -0,0 +1,9 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'tfctl/config.rb'
4
+ require_relative 'tfctl/generator.rb'
5
+ require_relative 'tfctl/executor.rb'
6
+ require_relative 'tfctl/error.rb'
7
+ require_relative 'tfctl/logger.rb'
8
+ require_relative 'tfctl/aws_org.rb'
9
+ require_relative 'tfctl/version.rb'
data/tfctl.gemspec ADDED
@@ -0,0 +1,29 @@
1
+ # frozen_string_literal: true
2
+ $LOAD_PATH << File.expand_path("../lib", __FILE__)
3
+ require 'tfctl/version'
4
+
5
+ Gem::Specification.new do |spec|
6
+ spec.name = 'tfctl'
7
+ spec.version = Tfctl::VERSION
8
+ spec.authors = [
9
+ 'Andrew Wasilczuk'
10
+ ]
11
+ spec.email = [
12
+ 'akw@scalefactory.com'
13
+ ]
14
+ spec.summary = 'Terraform wrapper for managing multi-account AWS infrastructures'
15
+ spec.homepage = 'https://github.com/scalefactory/tfctl'
16
+ spec.license = "MIT"
17
+ spec.files = `git ls-files -z`.split("\x0").reject { |f|
18
+ f.match(%r{^(test|spec|features)/})
19
+ }
20
+ spec.bindir = "bin"
21
+ spec.executables = spec.files.grep(%r{^bin/tfctl}) { |f| File.basename(f) }
22
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
23
+ spec.require_paths = ["lib"]
24
+
25
+ spec.add_dependency 'aws-sdk-organizations', '~> 1.13'
26
+ spec.add_dependency 'parallel', '~> 1.17'
27
+
28
+ spec.add_development_dependency 'rspec', '~> 3.8'
29
+ end
metadata ADDED
@@ -0,0 +1,119 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: tfctl
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.2
5
+ platform: ruby
6
+ authors:
7
+ - Andrew Wasilczuk
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2019-11-08 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: aws-sdk-organizations
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '1.13'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '1.13'
27
+ - !ruby/object:Gem::Dependency
28
+ name: parallel
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '1.17'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '1.17'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rspec
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '3.8'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '3.8'
55
+ description:
56
+ email:
57
+ - akw@scalefactory.com
58
+ executables:
59
+ - tfctl
60
+ extensions: []
61
+ extra_rdoc_files: []
62
+ files:
63
+ - ".gitignore"
64
+ - ".rspec"
65
+ - ".travis.yml"
66
+ - CHANGELOG.adoc
67
+ - Gemfile
68
+ - LICENSE
69
+ - Makefile
70
+ - README.adoc
71
+ - bin/tfctl
72
+ - docs/configuration.adoc
73
+ - docs/control_tower.adoc
74
+ - docs/creating_a_profile.adoc
75
+ - docs/iam_permissions.adoc
76
+ - docs/project_layout.adoc
77
+ - examples/bootstrap/terraform-exec-role.template
78
+ - examples/bootstrap/terraform-state.template
79
+ - examples/bootstrap/tfctl-org-access.template
80
+ - examples/control_tower/conf/example.yaml
81
+ - examples/control_tower/modules/s3-bucket/main.tf
82
+ - examples/control_tower/modules/s3-bucket/variables.tf
83
+ - examples/control_tower/profiles/example-profile/data.tf
84
+ - examples/control_tower/profiles/example-profile/main.tf
85
+ - examples/control_tower/profiles/example-profile/variables.tf
86
+ - lib/hash.rb
87
+ - lib/tfctl.rb
88
+ - lib/tfctl/aws_org.rb
89
+ - lib/tfctl/config.rb
90
+ - lib/tfctl/error.rb
91
+ - lib/tfctl/executor.rb
92
+ - lib/tfctl/generator.rb
93
+ - lib/tfctl/logger.rb
94
+ - lib/tfctl/version.rb
95
+ - tfctl.gemspec
96
+ homepage: https://github.com/scalefactory/tfctl
97
+ licenses:
98
+ - MIT
99
+ metadata: {}
100
+ post_install_message:
101
+ rdoc_options: []
102
+ require_paths:
103
+ - lib
104
+ required_ruby_version: !ruby/object:Gem::Requirement
105
+ requirements:
106
+ - - ">="
107
+ - !ruby/object:Gem::Version
108
+ version: '0'
109
+ required_rubygems_version: !ruby/object:Gem::Requirement
110
+ requirements:
111
+ - - ">="
112
+ - !ruby/object:Gem::Version
113
+ version: '0'
114
+ requirements: []
115
+ rubygems_version: 3.0.3
116
+ signing_key:
117
+ specification_version: 4
118
+ summary: Terraform wrapper for managing multi-account AWS infrastructures
119
+ test_files: []