dh-proteus 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +21 -0
  3. data/.rspec +3 -0
  4. data/.travis.yml +7 -0
  5. data/Gemfile +4 -0
  6. data/Gemfile.lock +80 -0
  7. data/README.md +414 -0
  8. data/Rakefile +6 -0
  9. data/bin/console +14 -0
  10. data/bin/proteus +5 -0
  11. data/bin/proteus_testing +8 -0
  12. data/bin/setup +8 -0
  13. data/build.sh +28 -0
  14. data/dh-proteus.gemspec +50 -0
  15. data/lib/core_ext/hash.rb +14 -0
  16. data/lib/proteus.rb +9 -0
  17. data/lib/proteus/app.rb +86 -0
  18. data/lib/proteus/backend/backend.rb +41 -0
  19. data/lib/proteus/commands/apply.rb +53 -0
  20. data/lib/proteus/commands/clean.rb +22 -0
  21. data/lib/proteus/commands/destroy.rb +51 -0
  22. data/lib/proteus/commands/graph.rb +22 -0
  23. data/lib/proteus/commands/import.rb +75 -0
  24. data/lib/proteus/commands/move.rb +28 -0
  25. data/lib/proteus/commands/output.rb +29 -0
  26. data/lib/proteus/commands/plan.rb +55 -0
  27. data/lib/proteus/commands/remove.rb +71 -0
  28. data/lib/proteus/commands/render.rb +36 -0
  29. data/lib/proteus/commands/taint.rb +35 -0
  30. data/lib/proteus/common.rb +72 -0
  31. data/lib/proteus/config/config.rb +47 -0
  32. data/lib/proteus/context_management/context.rb +31 -0
  33. data/lib/proteus/context_management/helpers.rb +14 -0
  34. data/lib/proteus/generate.rb +18 -0
  35. data/lib/proteus/generators/context.rb +57 -0
  36. data/lib/proteus/generators/environment.rb +42 -0
  37. data/lib/proteus/generators/init.rb +40 -0
  38. data/lib/proteus/generators/module.rb +69 -0
  39. data/lib/proteus/generators/templates/config/config.yaml.erb +22 -0
  40. data/lib/proteus/generators/templates/context/main.tf.erb +7 -0
  41. data/lib/proteus/generators/templates/context/variables.tf.erb +3 -0
  42. data/lib/proteus/generators/templates/environment/terraform.tfvars.erb +6 -0
  43. data/lib/proteus/generators/templates/module/io.tf.erb +1 -0
  44. data/lib/proteus/generators/templates/module/module.tf.erb +1 -0
  45. data/lib/proteus/generators/templates/module/validator.rb.erb +9 -0
  46. data/lib/proteus/global_commands/validate.rb +45 -0
  47. data/lib/proteus/helpers.rb +91 -0
  48. data/lib/proteus/helpers/path_helpers.rb +90 -0
  49. data/lib/proteus/helpers/string_helpers.rb +13 -0
  50. data/lib/proteus/init.rb +16 -0
  51. data/lib/proteus/modules/manager.rb +53 -0
  52. data/lib/proteus/modules/terraform_module.rb +184 -0
  53. data/lib/proteus/templates/partial.rb +123 -0
  54. data/lib/proteus/templates/template_binding.rb +62 -0
  55. data/lib/proteus/validators/base_validator.rb +24 -0
  56. data/lib/proteus/validators/validation_dsl.rb +172 -0
  57. data/lib/proteus/validators/validation_error.rb +9 -0
  58. data/lib/proteus/validators/validation_helpers.rb +9 -0
  59. data/lib/proteus/version.rb +7 -0
  60. metadata +260 -0
@@ -0,0 +1,13 @@
1
+ module Proteus
2
+ module Helpers
3
+ module StringHelpers
4
+ def camel_case(input)
5
+ input.split('_').collect(&:capitalize).join
6
+ end
7
+
8
+ def _(input)
9
+ input.gsub('-', '_')
10
+ end
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,16 @@
1
+ require 'proteus/generators/init'
2
+
3
+ module Proteus
4
+ class Init < Thor
5
+ include Thor::Actions
6
+ include Helpers
7
+
8
+ def self.source_root
9
+ File.expand_path(File.join(File.dirname(__FILE__), 'generators', 'templates'))
10
+ end
11
+
12
+ include Generators::Init
13
+
14
+ default_task :init
15
+ end
16
+ end
@@ -0,0 +1,53 @@
1
+ require 'proteus/helpers/path_helpers'
2
+ require 'proteus/modules/terraform_module'
3
+ require 'proteus/backend/backend'
4
+
5
+ module Proteus
6
+ module Modules
7
+
8
+ class Manager
9
+ include Thor::Shell
10
+ include Proteus::Helpers::PathHelpers
11
+
12
+ def initialize(context:, environment:)
13
+ @context = context
14
+ @environment = environment
15
+ initialize_modules
16
+ end
17
+
18
+ def render_modules
19
+ @modules.map(&:process)
20
+ end
21
+
22
+ def clean
23
+ @modules.map(&:clean)
24
+ end
25
+
26
+ private
27
+
28
+ def initialize_modules
29
+ say "Initializing modules", :green
30
+
31
+ tfvars_content = File.read(File.join(environments_path(@context), "terraform.#{@environment}.tfvars"))
32
+
33
+ if tfvars_content.empty?
34
+ terraform_variables = []
35
+ else
36
+ terraform_variables = HCL::Checker.parse(tfvars_content).with_indifferent_access
37
+ end
38
+
39
+ @modules = []
40
+
41
+ Dir.glob("#{modules_path(@context)}/*").each do |directory|
42
+ @modules << TerraformModule.new(
43
+ name: File.basename(directory),
44
+ context: @context,
45
+ environment: @environment,
46
+ terraform_variables: terraform_variables
47
+ )
48
+ end
49
+ end
50
+
51
+ end
52
+ end
53
+ end
@@ -0,0 +1,184 @@
1
+ require 'proteus/helpers/path_helpers'
2
+ require 'proteus/helpers/string_helpers'
3
+ require 'hcl/checker'
4
+ require 'yaml'
5
+
6
+ module Proteus
7
+ module Modules
8
+ class TerraformModule
9
+ @@hooks = {}
10
+
11
+ include Thor::Shell
12
+ extend Thor::Shell
13
+ include Proteus::Helpers::PathHelpers
14
+ extend Proteus::Helpers::PathHelpers
15
+ include Proteus::Helpers::StringHelpers
16
+
17
+ attr_reader :name
18
+
19
+ def initialize(name:, context:, environment:, terraform_variables:)
20
+ @name = name
21
+ @context = context
22
+ @environment = environment
23
+ @terraform_variables = terraform_variables
24
+ @hook_variables = {}
25
+ end
26
+
27
+ def process
28
+ clean
29
+ run_hooks
30
+ load_data
31
+ return if !data?
32
+ validate
33
+ render
34
+ end
35
+
36
+ def clean
37
+ File.file?(manifest) ? FileUtils.rm(manifest) : nil
38
+ end
39
+
40
+ def self.register_hook(module_name, hook)
41
+ @@hooks[module_name] ||= Array.new
42
+ @@hooks[module_name] << hook
43
+ end
44
+
45
+ def run_hooks
46
+ Dir.glob(File.join(module_hooks_path(@context, @name), '*')).each do |file|
47
+ require file
48
+ end
49
+
50
+ hooks_module = "Proteus::Modules::#{camel_case(@name)}::Hooks"
51
+ if Kernel.const_defined?(hooks_module)
52
+ say "Hooks present for module #{@name}", :green
53
+ Kernel.const_get(hooks_module).constants.each do |constant|
54
+ say "Found hook: #{constant}", :green
55
+ self.class.include(Kernel.const_get("#{hooks_module}::#{constant}"))
56
+ end
57
+ end
58
+
59
+ if @@hooks.key?(@name)
60
+ @@hooks[@name].each do |hook|
61
+ hook_result = hook.call(@environment, @context, @name)
62
+
63
+ if hook_result.is_a?(Hash)
64
+ @hook_variables.merge!(hook_result)
65
+ end
66
+
67
+ @@hooks[@name].delete(hook)
68
+ end
69
+ end
70
+ end
71
+
72
+ private
73
+
74
+
75
+ def load_data
76
+ @data = {}
77
+
78
+ if File.file?(config_file)
79
+ @data = YAML.load_file(config_file, {}).with_indifferent_access
80
+ end
81
+ end
82
+
83
+ def data?
84
+ return false unless @data
85
+ @data.any?
86
+ end
87
+
88
+ def validate
89
+ if data? && File.file?(validator_file)
90
+ require validator_file
91
+
92
+ begin
93
+ validator_class = "Proteus::Validators::#{camel_case(@name)}Validator"
94
+
95
+ Kernel.const_get(validator_class).new(@data, @environment)
96
+
97
+ say "Ran #{camel_case(@name)}Validator for environment #{@environment} #{"\u2714".encode('utf-8')}", :green
98
+ rescue Proteus::Validators::ValidationError => validation_error
99
+ say "#{validator_class}: #{validation_error.message} [modules/#{@name}/config/#{@environment}.yaml] #{"\u2718".encode('utf-8')}", :red
100
+ exit 1
101
+ end
102
+ else
103
+ say "Module #{@name} has no validator.", :magenta
104
+ end
105
+ end
106
+
107
+ def render
108
+ File.open(manifest, "w") do |manifest|
109
+ manifest << global_resources
110
+
111
+ @data.fetch('template_data', {}).each do |template, data|
112
+ manifest << render_template(File.join(module_templates_path(@context, @name), "#{template}.tf.erb"), data)
113
+ end
114
+ end
115
+ end
116
+
117
+ def render_template(template_file, data)
118
+ if File.file?(template_file)
119
+ template = File.read(template_file)
120
+ begin
121
+ return "#{Erubis::Eruby.new(template).result(template_binding(data).get_binding)}\n\n"
122
+ rescue Exception => e
123
+ say "Error in template: #{template_file}", :magenta
124
+ e.backtrace.each { |line| say line, :magenta }
125
+ say e.message, :magenta
126
+ exit 1
127
+ end
128
+ end
129
+ end
130
+
131
+ def template_binding(data)
132
+ binding = Proteus::Templates::TemplateBinding.new(
133
+ context: @context,
134
+ environment: @environment,
135
+ module_name: @name,
136
+ data: data,
137
+ defaults: parse_defaults
138
+ )
139
+
140
+ binding.set(:terraform_variables, @terraform_variables)
141
+
142
+ @hook_variables.each do |key, value|
143
+ binding.set(key, value)
144
+ end
145
+
146
+ binding
147
+ end
148
+
149
+ def parse_defaults
150
+ defaults = []
151
+
152
+ return defaults unless File.file?(File.join(module_path(@context, @name), 'io.tf'))
153
+ HCL::Checker.parse(File.read(File.join(module_path(@context, @name), 'io.tf')))['variable'].each do |variable, values|
154
+ if values
155
+ defaults.push(variable) if values.has_key?('default')
156
+ end
157
+ end
158
+ return defaults
159
+ end
160
+
161
+ def global_resources
162
+ Dir.glob(File.join(module_config_path(@context, @name), 'global_resources/*')).inject("") do |memo, resource_file|
163
+ if resource_file =~ /tf\.erb/
164
+ "#{memo}\n#{render_template(resource_file, @data.dig('global_resources', File.basename(resource_file, '.tf.erb')))}"
165
+ else
166
+ "#{memo}#{File.read(resource_file)}\n"
167
+ end
168
+ end
169
+ end
170
+
171
+ def config_file
172
+ File.join(module_config_path(@context, @name), "#{@environment}.yaml")
173
+ end
174
+
175
+ def validator_file
176
+ File.join(module_config_path(@context, @name), 'validator.rb')
177
+ end
178
+
179
+ def manifest
180
+ File.join(context_root_path(@context), "#{@name}.tf")
181
+ end
182
+ end
183
+ end
184
+ end
@@ -0,0 +1,123 @@
1
+ require 'hcl/checker'
2
+
3
+ module Proteus
4
+ module Templates
5
+ class Partial < TemplateBinding
6
+
7
+ include Proteus::Helpers::PathHelpers
8
+ include Thor::Shell
9
+
10
+ def initialize(name:, context:, environment:, module_name:, data:, data_context: nil, force_rendering: true, deep_merge: false, terraform_variables:, scope_resources: nil)
11
+ @name = name
12
+ @context = context
13
+ @environment = environment
14
+ @module_name = module_name
15
+ @data = data
16
+ @data_context = data_context
17
+ @force_rendering = force_rendering
18
+ @deep_merge = deep_merge
19
+ @terraform_variables = terraform_variables
20
+ @scope_resources = scope_resources
21
+
22
+ @defaults = {}
23
+ set(@name.split('/').last, {})
24
+
25
+ if data.dig('partials', @name)
26
+ @partial_data_present = true
27
+
28
+ @data['partials'][@name].delete('render')
29
+
30
+ set(@name, @data['partials'][@name])
31
+ end
32
+
33
+ end
34
+
35
+ def render
36
+ if partial_data? || @force_rendering
37
+ defaults_file = File.join(module_templates_path(@context, @module_name), 'defaults', "#{@name}.yaml")
38
+
39
+ default_partial_path = File.join(module_templates_path(@context, @module_name), "_#{@name}.tf.erb")
40
+ sub_directory_partial_path = File.join(module_templates_path(@context, @module_name), "#{@name}.tf.erb")
41
+
42
+ if File.file?(default_partial_path)
43
+ partial_file = default_partial_path
44
+ else
45
+ partial_file = sub_directory_partial_path
46
+ end
47
+
48
+
49
+ if File.file?(partial_file)
50
+ partial_template = File.read(partial_file)
51
+
52
+ if File.file?(defaults_file)
53
+ @defaults = YAML.load_file(defaults_file, {}).with_indifferent_access
54
+
55
+ partial_data = instance_variable_get("@#{@name}")
56
+
57
+ if @deep_merge == false
58
+ @defaults.each do |key, value|
59
+ if !partial_data.key?(key)
60
+ partial_data[key] = value
61
+ end
62
+ end
63
+ elsif @deep_merge == :each
64
+ merged_data = []
65
+
66
+ partial_data.each do |item|
67
+ merged_data << @defaults.deep_merge(item)
68
+ end
69
+
70
+ instance_variable_set("@#{@name.split('/').last}", merged_data)
71
+ else
72
+ instance_variable_set("@#{@name.split('/').last}", @defaults.deep_merge(partial_data))
73
+ end
74
+
75
+ end
76
+
77
+ begin
78
+ if @scope_resources
79
+ return scope_resources(manifest: Erubis::Eruby.new(partial_template).result(get_binding), scope: @scope_resources)
80
+ else
81
+ return Erubis::Eruby.new(partial_template).result(get_binding)
82
+ end
83
+ rescue Exception => e
84
+ say "Error in partial: #{partial_file}", :magenta
85
+ e.backtrace.each { |line| say line, :magenta }
86
+ say e.message, :magenta
87
+ exit 1
88
+ end
89
+ end
90
+ end
91
+ end
92
+
93
+ def partial_data?
94
+ @partial_data_present
95
+ end
96
+
97
+ def scope_resources(manifest:, scope:)
98
+ scoped = []
99
+ manifest.each_line do |line|
100
+ if matches = line.match(/resource( +)"(?<resource_type>[a-z0-9_]+)"( +)"(?<resource_name>[a-zA-Z0-9_\-]+)"( +)(\{)/)
101
+ say "MATCHED RESOURCE: #{line}", :green
102
+ matches = matches.named_captures.with_indifferent_access
103
+ scoped << "resource \"#{matches[:resource_type]}\" \"#{scope}_#{matches[:resource_name]}\" {"
104
+ say "CHANGED TO: #{scoped.last}", :green
105
+ elsif matches = line.match(/(?<pre>.*(\{|\())(?<data>data\.)?(?<resource_type>(?<provider>aws|kubernetes|helm|tls_private_key|null_resource)[a-z_0-9]*)\.(?<resource_name>[a-z0-9_]+)(?<attribute>(?<wildcard>.\*)?\.[a-z_]+)?(?<post>.*)?/)
106
+ matches = matches.named_captures.with_indifferent_access
107
+
108
+ if !matches[:data]
109
+ say "MATCHED REFERENCE: #{line}", :green
110
+ scoped << "#{matches[:pre]}#{matches[:resource_type]}.#{scope}_#{matches[:resource_name]}#{matches[:attribute]}#{matches[:post]}"
111
+ say "CHANGED TO: #{scoped.last}", :green
112
+ else
113
+ scoped << line
114
+ end
115
+ else
116
+ scoped << line
117
+ end
118
+ end
119
+ scoped.map!(&:chomp).join("\n")
120
+ end
121
+ end
122
+ end
123
+ end
@@ -0,0 +1,62 @@
1
+ module Proteus
2
+ module Templates
3
+ class TemplateBinding
4
+
5
+ include Proteus::Helpers::PathHelpers
6
+ include Proteus::Helpers::StringHelpers
7
+
8
+ def initialize(context:, environment:, module_name:, data: {}, defaults: [])
9
+ @context = context
10
+ @environment = environment
11
+ @module_name = module_name
12
+
13
+ data.each do |key, value|
14
+ set(key, value)
15
+ end
16
+ @defaults = defaults
17
+ end
18
+
19
+ def set(name, value)
20
+ instance_variable_set("@#{name}", value)
21
+ end
22
+
23
+ def get_binding
24
+ binding
25
+ end
26
+
27
+ def render_defaults(context, demo: false)
28
+ @defaults.inject("") do |memo, default|
29
+ if context.has_key?(default)
30
+ memo << "#{demo ? "# " : ""}#{default} = \"#{context[default]}\"\n"
31
+ else
32
+ memo
33
+ end
34
+ end
35
+ end
36
+
37
+ # return output of partial template
38
+ # name: symbol (template_name)
39
+ def render_partial(name:, data:, data_context: nil, force_rendering: true, deep_merge: false, scope_resources: nil)
40
+ partial = Partial.new(
41
+ name: name.to_s,
42
+ context: @context,
43
+ environment: @environment,
44
+ module_name: @module_name,
45
+ data: data,
46
+ data_context: data_context,
47
+ force_rendering: force_rendering,
48
+ deep_merge: deep_merge,
49
+ terraform_variables: @terraform_variables,
50
+ scope_resources: scope_resources
51
+ )
52
+
53
+ partial.render
54
+ end
55
+
56
+ def default_true(context, key)
57
+ return context.has_key?(key) ? context[key] : true
58
+ end
59
+ end
60
+ end
61
+ end
62
+