cloudshaper 0.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +4 -0
  3. data/.travis.yml +3 -0
  4. data/Gemfile +7 -0
  5. data/Gemfile.lock +27 -0
  6. data/LICENSE +23 -0
  7. data/README.md +162 -0
  8. data/Rakefile +9 -0
  9. data/TODO.md +7 -0
  10. data/examples/secretconfig/README.md +3 -0
  11. data/examples/secretconfig/atlas.json +5 -0
  12. data/examples/secretconfig/aws.json +6 -0
  13. data/examples/secretconfig/cloudflare.json +6 -0
  14. data/examples/secretconfig/cloudstack.json +7 -0
  15. data/examples/secretconfig/digitalocean.json +5 -0
  16. data/examples/secretconfig/dme.json +6 -0
  17. data/examples/secretconfig/heroku.json +6 -0
  18. data/examples/secretconfig/mailgun.json +5 -0
  19. data/examples/simple_app.rb +61 -0
  20. data/lib/tasks/tasks.rb +12 -0
  21. data/lib/tasks/terraform.rake +127 -0
  22. data/lib/terraform_dsl.rb +6 -0
  23. data/lib/terraform_dsl/aws/remote_s3.rb +20 -0
  24. data/lib/terraform_dsl/aws/tagging.rb +23 -0
  25. data/lib/terraform_dsl/command.rb +58 -0
  26. data/lib/terraform_dsl/module.rb +38 -0
  27. data/lib/terraform_dsl/output.rb +6 -0
  28. data/lib/terraform_dsl/provider.rb +13 -0
  29. data/lib/terraform_dsl/remote.rb +31 -0
  30. data/lib/terraform_dsl/resource.rb +44 -0
  31. data/lib/terraform_dsl/secrets.rb +29 -0
  32. data/lib/terraform_dsl/stack.rb +79 -0
  33. data/lib/terraform_dsl/stack_element.rb +62 -0
  34. data/lib/terraform_dsl/stack_module.rb +121 -0
  35. data/lib/terraform_dsl/stack_modules.rb +24 -0
  36. data/lib/terraform_dsl/stacks.rb +64 -0
  37. data/lib/terraform_dsl/variable.rb +6 -0
  38. data/lib/terraform_dsl/version.rb +3 -0
  39. data/terraform_dsl.gemspec +25 -0
  40. data/test/stack_module_test.rb +152 -0
  41. data/test/stack_test.rb +5 -0
  42. data/test/test_helper.rb +4 -0
  43. metadata +131 -0
@@ -0,0 +1,6 @@
1
+ require 'terraform_dsl/stack'
2
+ require 'terraform_dsl/stacks'
3
+ require 'terraform_dsl/stack_module'
4
+ require 'terraform_dsl/version'
5
+
6
+ require 'tasks/tasks' if defined? Rake
@@ -0,0 +1,20 @@
1
+ module Terraform
2
+ module Aws
3
+ # Support AWS S3 remote state backend
4
+ module RemoteS3
5
+ private
6
+
7
+ def options_for_s3(command)
8
+ options = ''
9
+ options = "-backend=s3 #{config_opts_s3}" if command == :config
10
+ "terraform remote #{command} #{options}"
11
+ end
12
+
13
+ def config_opts_s3
14
+ config = "-backend-config='key=#{@stack.stack_id}' "
15
+ config += @stack.remote['s3'].map { |k, v| "-backend-config='#{k}=#{v}'" }.join(' ')
16
+ config
17
+ end
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,23 @@
1
+ module Terraform
2
+ # Aws provider-specific functionality, to be mixed in to stack elements
3
+ module Aws
4
+ def self.taggable?(resource_type)
5
+ supports_tagging = [:aws_autoscaling_group, :aws_customer_gateway,
6
+ :aws_db_instance, :aws_elasticache_cluster,
7
+ :aws_elb, :aws_instance, :aws_internet_gateway,
8
+ :aws_network_acl, :aws_network_interface, :aws_route53_zone,
9
+ :aws_route_table, :aws_s3_bucket, :aws_security_group,
10
+ :aws_subnet, :aws_vpc, :aws_vpc_dhcp_options,
11
+ :aws_vpc_peering_connection, :aws_vpn_connection,
12
+ :aws_vpn_gateway]
13
+ supports_tagging.include?(resource_type.to_sym)
14
+ end
15
+
16
+ # Tag all resources (that support tagging) that we created with this stack id
17
+ def post_processing_aws
18
+ return unless Aws.taggable?(@resource_type)
19
+ @fields[:tags] ||= {}
20
+ @fields[:tags][:terraform_stack_id] = var(:terraform_stack_id)
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,58 @@
1
+ module Terraform
2
+ # Wraps terraform command execution
3
+ class Command
4
+ attr_accessor :command
5
+
6
+ def initialize(stack, command)
7
+ @stack = stack
8
+ @command = options_for(command)
9
+ prepare
10
+ end
11
+
12
+ def env
13
+ vars = {}
14
+ @stack.variables.each { |k, v| vars["TF_VAR_#{k}"] = v[:default] }
15
+ @stack.module.secrets.each do |_provider, secrets|
16
+ secrets.each do |k, v|
17
+ vars[k.to_s] = v
18
+ end
19
+ end
20
+ vars
21
+ end
22
+
23
+ def execute
24
+ Process.waitpid(spawn(env, @command, chdir: @stack.stack_dir))
25
+ fail 'Command failed' unless $CHILD_STATUS.to_i == 0
26
+ end
27
+
28
+ protected
29
+
30
+ def prepare
31
+ FileUtils.mkdir_p(@stack.stack_dir)
32
+ File.open(File.join(@stack.stack_dir, 'terraform.tf.json'), 'w') { |f| f.write(generate) }
33
+ end
34
+
35
+ def options_for(cmd)
36
+ options = begin
37
+ case cmd
38
+ when :apply
39
+ '-input=false'
40
+ when :destroy
41
+ '-input=false -force'
42
+ when :plan
43
+ '-input=false -module-depth=-1'
44
+ when :graph
45
+ '-draw-cycles'
46
+ else
47
+ ''
48
+ end
49
+ end
50
+
51
+ "terraform #{cmd} #{options}"
52
+ end
53
+
54
+ def generate
55
+ @stack.module.generate
56
+ end
57
+ end
58
+ end
@@ -0,0 +1,38 @@
1
+ require 'terraform_dsl/stack_element'
2
+ require 'terraform_dsl/stack_modules'
3
+ require 'terraform_dsl/stack_module'
4
+ require 'terraform_dsl/stacks'
5
+
6
+ module Terraform
7
+ # Supports terraform 'modules'. In our case, we call them submodules because
8
+ # Module is a ruby keyword. We also support directly referencing other ruby-defined modules.
9
+ class Module < StackElement
10
+ def initialize(parent_module, module_name, &block)
11
+ super(parent_module, &block)
12
+
13
+ if @fields[:source].match(/^module_/)
14
+ build_submodule(parent_module, module_name)
15
+ end
16
+ end
17
+
18
+ private
19
+
20
+ def build_submodule(parent_module, module_name)
21
+ generated = generate_child_module(parent_module)
22
+ module_path = File.join(Stacks.dir, parent_module.id, module_name.to_s)
23
+ FileUtils.mkdir_p(module_path)
24
+ File.open(File.join(module_path, 'stack_module.tf.json'), 'w') { |f| f.write(generated) }
25
+ @fields[:source] = File.expand_path(module_path)
26
+ end
27
+
28
+ def generate_child_module(parent_module)
29
+ variables = @fields.clone
30
+ variables.delete(:source)
31
+ variables[:terraform_stack_id] = parent_module.id
32
+ child_name = @fields[:source].gsub(/^module_/, '')
33
+ child_module = StackModules.get(child_name)
34
+ child_module.build(variables)
35
+ child_module.generate
36
+ end
37
+ end
38
+ end
@@ -0,0 +1,6 @@
1
+ require 'terraform_dsl/stack_element'
2
+
3
+ module Terraform
4
+ class Output < StackElement
5
+ end
6
+ end
@@ -0,0 +1,13 @@
1
+ require 'terraform_dsl/secrets'
2
+ require 'terraform_dsl/stack_element'
3
+
4
+ module Terraform
5
+ # Implements DSL for a terraform provider, and a means of loading secrets.
6
+ class Provider < StackElement
7
+ def load_secrets(name)
8
+ @secrets ||= {}
9
+ @secrets[name.to_sym] = SECRETS[name.to_sym]
10
+ @secrets
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,31 @@
1
+ require 'terraform_dsl/aws/remote_s3'
2
+
3
+ module Terraform
4
+ # Wrap 'remote' commands, such as config, pull, and push
5
+ # This allows us to store state remotely using different providers
6
+ class Remote < Command
7
+ class RemoteNotSupported < Exception; end
8
+ include Aws::RemoteS3
9
+ def initialize(stack, command)
10
+ super
11
+ unless @stack.remote.first
12
+ puts "\tWARNING: #{@stack.name} is not configured with a remote backend"
13
+ return
14
+ end
15
+
16
+ backend = @stack.remote.keys.first
17
+ sym = "options_for_#{backend}"
18
+
19
+ if self.respond_to?(sym, include_private: true)
20
+ @command = send(sym, command)
21
+ else
22
+ fail RemoteNotSupported, "Remote backend #{backend} is not supported yet"
23
+ end
24
+ end
25
+
26
+ def execute
27
+ return unless @stack.remote.first
28
+ super
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,44 @@
1
+ require 'terraform_dsl/stack_element'
2
+
3
+ module Terraform
4
+ # Defines a terraform resource
5
+ class Resource < StackElement
6
+ attr_reader :resource_name
7
+
8
+ def initialize(parent_module, resource_name, resource_type, &block)
9
+ @resource_name = resource_name
10
+ @resource_type = resource_type
11
+ super(parent_module, &block)
12
+
13
+ # Allow provider specific post processing
14
+ sym = "post_processing_#{resource_type.split('_').first}"
15
+ send(sym) if self.respond_to?(sym, include_private: true)
16
+ end
17
+
18
+ # Allow provisioner blocks to be nested within resources
19
+ def provisioner(provisioner_type, &block)
20
+ provisioner_type = provisioner_type.to_sym
21
+
22
+ @fields[:provisioner] = @fields[:provisioner] || []
23
+
24
+ provisioner_set = Provisioner.new(@module, &block)
25
+ @fields[:provisioner] << { cleanup_provisioner_type(provisioner_type) => provisioner_set.fields }
26
+ end
27
+
28
+ private
29
+
30
+ def cleanup_provisioner_type(provisioner_type)
31
+ case provisioner_type.to_sym
32
+ when :remote_exec
33
+ 'remote-exec'
34
+ when :local_exec
35
+ 'local-exec'
36
+ else
37
+ provisioner_type
38
+ end
39
+ end
40
+ end
41
+ # Defines a terraform resource provisioner
42
+ class Provisioner < StackElement
43
+ end
44
+ end
@@ -0,0 +1,29 @@
1
+ require 'json'
2
+ require 'open3'
3
+
4
+ # Load and provide access to secrets required by terraform providers
5
+ class SecretHash < Hash
6
+ class SecretNotFound < Exception; end
7
+
8
+ def initialize
9
+ super { |_secrets, key| fail SecretNotFound, "Secret `#{key}` not found" }
10
+ end
11
+ end
12
+
13
+ if ENV['TERRAFORM_ENV'] == 'test'
14
+ SECRETS ||= {
15
+ aws: {
16
+ access_key_id: 'foo',
17
+ secret_access_key: 'bar'
18
+ }
19
+ }
20
+ else
21
+ SECRETS ||= begin
22
+ secrets_file = File.expand_path(ENV['CONFIG_PATH'] || 'config/secrets.json')
23
+ if File.exist?(secrets_file)
24
+ JSON.parse(File.read(secrets_file), symbolize_names: true, object_class: SecretHash)
25
+ else
26
+ {}
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,79 @@
1
+ require 'terraform_dsl/stacks'
2
+ require 'terraform_dsl/stack_modules'
3
+ require 'terraform_dsl/command'
4
+ require 'terraform_dsl/remote'
5
+
6
+ module Terraform
7
+ # Wrapper to instantiate a stack from a yaml definition
8
+ class Stack
9
+ class MalformedConfig < Exception; end
10
+ class << self
11
+ def load(config)
12
+ fail MalformedConfig, "Configuration malformed at #{config}" unless config.is_a?(Hash)
13
+ fail MalformedConfig, "A name must be specified for the stack #{config}" unless config.key?('name')
14
+ fail MalformedConfig, 'You must specify a uuid. Get one from rake uuid and add it to the config' unless config.key?('uuid')
15
+ new(config)
16
+ end
17
+ end
18
+
19
+ attr_reader :name, :description, :module,
20
+ :stack_dir, :stack_id, :remote
21
+
22
+ def initialize(config)
23
+ @name = config['name']
24
+ @uuid = config['uuid']
25
+ @description = config['description'] || ''
26
+ @variables = config['variables'] || {}
27
+ @remote = config['remote'] || {}
28
+ @stack_id = "terraform_#{@name}_#{@uuid}"
29
+ @module = StackModules.get(config['root'])
30
+ @variables['terraform_stack_id'] = @stack_id
31
+ @stack_dir = File.join(Stacks.dir, @stack_id)
32
+ @module.build(@variables.map { |k, v| [k.to_sym, v] }.to_h)
33
+ end
34
+
35
+ def apply
36
+ Command.new(self, :apply).execute
37
+ end
38
+
39
+ def destroy
40
+ Command.new(self, :destroy).execute
41
+ end
42
+
43
+ def plan
44
+ Command.new(self, :plan).execute
45
+ end
46
+
47
+ def get
48
+ Command.new(self, :get).execute
49
+ end
50
+
51
+ def show
52
+ Command.new(self, :show).execute
53
+ end
54
+
55
+ def pull
56
+ Remote.new(self, :pull).execute
57
+ end
58
+
59
+ def push
60
+ Remote.new(self, :pull).execute
61
+ end
62
+
63
+ def remote_config
64
+ Remote.new(self, :config).execute
65
+ end
66
+
67
+ def variables
68
+ @module.variables
69
+ end
70
+
71
+ def to_s
72
+ <<-eos
73
+ Name: #{@name}
74
+ Description: #{@description}
75
+ Stack Directory: #{@stack_dir}
76
+ eos
77
+ end
78
+ end
79
+ end
@@ -0,0 +1,62 @@
1
+ require 'terraform_dsl/aws/tagging'
2
+
3
+ module Terraform
4
+ # Defines a single terraform stack element, subclass for any element defined in terraform DSL
5
+ class StackElement
6
+ include Aws
7
+ attr_reader :fields
8
+
9
+ def initialize(stack_module, &block)
10
+ @module = stack_module
11
+ @fields = {}
12
+ instance_eval(&block)
13
+ end
14
+
15
+ private
16
+
17
+ # Allows resource attributes to be specified with a nice syntax
18
+ # If the method is implemented, it will be treated as a nested resource
19
+ def method_missing(method_name, *args, &block)
20
+ symbol = method_name.to_sym
21
+ if args.length == 1
22
+ if args[0].nil?
23
+ fail "Passed nil to '#{method_name}'. Generally disallowed, subclass StackElement if you need this."
24
+ end
25
+ add_field(symbol, args[0])
26
+ else
27
+ add_field(symbol, Terraform::StackElement.new(@module, &block).fields)
28
+ end
29
+ end
30
+
31
+ # Get the runtime value of a variable
32
+ def get(variable_name)
33
+ @module.get(variable_name)
34
+ end
35
+
36
+ # Reference a variable
37
+ def var(variable_name)
38
+ "${var.#{variable_name}}"
39
+ end
40
+
41
+ # Syntax to handle interpolation of resource variables
42
+ def value_of(resource_type, resource_name, value_type)
43
+ "${#{resource_type}.#{resource_name}.#{value_type}}"
44
+ end
45
+
46
+ # Shorthand to interpolate the ID of another resource
47
+ def id_of(resource_type, resource_name)
48
+ value_of(resource_type, resource_name, :id)
49
+ end
50
+
51
+ def add_field(symbol, value)
52
+ existing = @fields[symbol]
53
+ if existing
54
+ # If it's already an array, just push to it
55
+ @fields[symbol] = [existing] unless existing.is_a?(Array)
56
+ @fields[symbol] << value
57
+ else
58
+ @fields[symbol] = value
59
+ end
60
+ end
61
+ end
62
+ end
@@ -0,0 +1,121 @@
1
+ require 'json'
2
+ require 'fileutils'
3
+
4
+ require 'terraform_dsl/stack_modules'
5
+ require 'terraform_dsl/resource'
6
+ require 'terraform_dsl/provider'
7
+ require 'terraform_dsl/variable'
8
+ require 'terraform_dsl/module'
9
+ require 'terraform_dsl/output'
10
+
11
+ module Terraform
12
+ # Stack Modules contain stack elements. A stack is made up of a root module, which may have submodules
13
+ class StackModule
14
+ class << self
15
+ def define(name, &block)
16
+ template = new(name, &block)
17
+ StackModules.register(name, template)
18
+ end
19
+
20
+ def flatten_variable_arrays(variables)
21
+ vars = variables.map do |k, v|
22
+ if v.is_a?(Hash) && v.key?(:default) && v[:default].is_a?(Array)
23
+ v[:default] = v[:default].join(',')
24
+ elsif v.is_a?(Array)
25
+ v = v.join(',')
26
+ end
27
+ [k, v]
28
+ end
29
+ Hash[vars]
30
+ end
31
+ end
32
+
33
+ attr_accessor :secrets
34
+
35
+ def initialize(_name, &block)
36
+ @stack_elements = { resource: {}, provider: {}, variable: {}, output: {}, module: {} }
37
+ @secrets = {}
38
+ @block = block
39
+ variable(:terraform_stack_id) {}
40
+ end
41
+
42
+ def clone
43
+ b = @block
44
+ StackModule.new(@name, &b)
45
+ end
46
+
47
+ def build(**kwargs)
48
+ vars = Hash[kwargs.map { |k, v| [k, { default: v }] }]
49
+ @stack_elements[:variable].merge!(vars)
50
+ b = @block
51
+ instance_eval(&b)
52
+ end
53
+
54
+ def generate
55
+ JSON.pretty_generate(elements)
56
+ end
57
+
58
+ def variables
59
+ elements[:variable]
60
+ end
61
+
62
+ def outputs
63
+ @stack_elements[:output]
64
+ end
65
+
66
+ def get(variable)
67
+ @stack_elements[:variable].fetch(variable)[:default]
68
+ end
69
+
70
+ def elements
71
+ elements = @stack_elements.clone
72
+ variables = StackModule.flatten_variable_arrays(@stack_elements[:variable])
73
+ @stack_elements[:module].each do |mod, data|
74
+ elements[:module][mod] = StackModule.flatten_variable_arrays(data)
75
+ end
76
+ elements[:variable] = variables
77
+ elements
78
+ end
79
+
80
+ def id
81
+ get(:terraform_stack_id)
82
+ end
83
+
84
+ private
85
+
86
+ def register_resource(resource_type, name, &block)
87
+ @stack_elements[:resource] ||= {}
88
+ @stack_elements[:resource][resource_type.to_sym] ||= {}
89
+ @stack_elements[:resource][resource_type.to_sym][name.to_sym] = Terraform::Resource.new(self, name, resource_type, &block).fields
90
+ end
91
+
92
+ def register_variable(name, &block)
93
+ new_variable = Terraform::Variable.new(self, &block).fields
94
+ unless @stack_elements[:variable].key?(name.to_sym)
95
+ @stack_elements[:variable][name.to_sym] = { default: new_variable[:default] || '' }
96
+ end
97
+ end
98
+
99
+ def register_output(name, &block)
100
+ new_output = Terraform::Output.new(self, &block).fields
101
+ @stack_elements[:output][name.to_sym] = new_output
102
+ end
103
+
104
+ def register_module(name, &block)
105
+ new_module = Terraform::Module.new(self, name, &block).fields
106
+ @stack_elements[:module][name.to_sym] = new_module
107
+ end
108
+
109
+ def register_provider(name, &block)
110
+ provider = Terraform::Provider.new(self, &block)
111
+ @secrets.merge!(provider.load_secrets(name))
112
+ @stack_elements[:provider][name.to_sym] = provider.fields
113
+ end
114
+
115
+ alias_method :resource, :register_resource
116
+ alias_method :variable, :register_variable
117
+ alias_method :provider, :register_provider
118
+ alias_method :output, :register_output
119
+ alias_method :submodule, :register_module
120
+ end
121
+ end