cfhighlander 0.2.0.alpha.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,306 @@
1
+ require 'cfndsl'
2
+ require 'erb'
3
+ require 'fileutils'
4
+ require 'cfndsl/globals'
5
+ require 'cfndsl/version'
6
+ require 'json'
7
+ require 'yaml'
8
+ require 'open-uri'
9
+ require 'net/http'
10
+ require 'net/https'
11
+ require 'highline/import'
12
+ require 'zip'
13
+ require_relative './util/zip.util'
14
+
15
+ module Highlander
16
+
17
+ module Compiler
18
+
19
+ class ComponentCompiler
20
+
21
+ @@global_extensions_paths = []
22
+
23
+ attr_accessor :workdir,
24
+ :component,
25
+ :compiled_subcomponents,
26
+ :component_name,
27
+ :config_output_location,
28
+ :dsl_output_location,
29
+ :cfn_output_location,
30
+ :cfn_template_paths,
31
+ :silent_mode,
32
+ :lambda_src_paths
33
+
34
+ def initialize(component)
35
+
36
+ @workdir = ENV['HIGHLANDER_WORKDIR']
37
+ @component = component
38
+ @sub_components = []
39
+ @component_name = component.highlander_dsl.name.downcase
40
+ @cfndsl_compiled = false
41
+ @config_compiled = false
42
+ @cfn_template_paths = []
43
+ @lambdas_processed = false
44
+ @silent_mode = false
45
+ @lambda_src_paths = []
46
+
47
+ if @@global_extensions_paths.empty?
48
+ global_extensions_folder = "#{File.dirname(__FILE__)}/../cfndsl_ext"
49
+ Dir["#{global_extensions_folder}/*.rb"].each { |f| @@global_extensions_paths << f }
50
+ end
51
+
52
+ @component.highlander_dsl.components.each do |sub_component|
53
+ sub_component_compiler = Highlander::Compiler::ComponentCompiler.new(sub_component.component_loaded)
54
+ sub_component_compiler.component_name = sub_component.name
55
+ @sub_components << sub_component_compiler
56
+ end
57
+ end
58
+
59
+ def silent_mode=(value)
60
+ @silent_mode = value
61
+ @sub_components.each { |scc| scc.silent_mode=value }
62
+ end
63
+
64
+ def compileCfnDsl(out_format)
65
+ processLambdas unless @lambdas_processed
66
+ writeConfig unless @config_written
67
+ dsl = @component.highlander_dsl
68
+ component_cfndsl = @component.cfndsl_content
69
+
70
+ @component.highlander_dsl.components.each { |sc|
71
+ sc.distribution_format = out_format
72
+ }
73
+
74
+ # indent component cfndsl
75
+ component_cfndsl.gsub!("\n", "\n\t")
76
+ component_cfndsl.gsub!("\r\n", "\r\n\t")
77
+ # render cfndsl
78
+ renderer = ERB.new(File.read("#{__dir__}/../templates/cfndsl.component.template.erb"))
79
+ cfn_template = renderer.result(OpenStruct.new({
80
+ 'dsl' => dsl,
81
+ 'component_cfndsl' => component_cfndsl,
82
+ 'component_requires' => (@@global_extensions_paths + @component.cfndsl_ext_files)
83
+ }).instance_eval { binding })
84
+
85
+ # write to output file
86
+ output_dir = "#{@workdir}/out/cfndsl"
87
+ @dsl_output_location = output_dir
88
+ output_path = "#{output_dir}/#{@component_name}.compiled.cfndsl.rb"
89
+ FileUtils.mkdir_p(output_dir) unless Dir.exist?(output_dir)
90
+ File.write(output_path, cfn_template)
91
+ puts "cfndsl template for #{dsl.name} written to #{output_path}"
92
+ @cfndsl_compiled_path = output_path
93
+
94
+ @sub_components.each { |subcomponent_compiler|
95
+ puts "Rendering sub-component cfndsl: #{subcomponent_compiler.component_name}"
96
+ subcomponent_compiler.compileCfnDsl out_format
97
+ }
98
+
99
+ @cfndsl_compiled = true
100
+
101
+ end
102
+
103
+ def compileCloudFormation(format = 'yaml')
104
+
105
+
106
+ #compile cfndsl templates first
107
+ compileCfnDsl format unless @cfndsl_compiled
108
+
109
+ dsl = @component.highlander_dsl
110
+ component_cfndsl = @component.cfndsl_content
111
+
112
+ # create out dir if not there
113
+ @cfn_output_location = "#{@workdir}/out/#{format}"
114
+ output_dir = @cfn_output_location
115
+ FileUtils.mkdir_p(output_dir) unless Dir.exist?(output_dir)
116
+
117
+ # write config
118
+ config_yaml_path = writeConfig
119
+
120
+
121
+ # compile templates
122
+ output_path = "#{output_dir}/#{component_name}.compiled.#{format}"
123
+ @cfn_template_paths << output_path
124
+ # configure cfndsl
125
+ cfndsl_opts = []
126
+ cfndsl_opts.push([:yaml, config_yaml_path])
127
+
128
+ # grab cfndsl model
129
+ model = CfnDsl.eval_file_with_extras(@cfndsl_compiled_path, cfndsl_opts, false)
130
+
131
+ # write resulting cloud formation template
132
+ if format == 'json'
133
+ output_content = JSON.pretty_generate(model)
134
+ elsif format == 'yaml'
135
+ output_content = JSON.parse(model.to_json).to_yaml
136
+ else
137
+ raise StandardError, "#{format} not supported for cfn generation"
138
+ end
139
+
140
+ File.write(output_path, output_content)
141
+ # `cfndsl #{@cfndsl_compiled_path} -p -f #{format} -o #{output_path} --disable-binding`
142
+ puts "CloudFormation #{format.upcase} template for #{dsl.name} written to #{output_path}"
143
+
144
+ # compile sub-component templates
145
+ @sub_components.each do |sub_component|
146
+ sub_component.compileCloudFormation format
147
+ @cfn_template_paths += sub_component.cfn_template_paths
148
+ @lambda_src_paths += sub_component.lambda_src_paths
149
+ end
150
+
151
+ end
152
+
153
+ def writeConfig(write_subcomponents_config = false)
154
+ @config_output_location = "#{@workdir}/out/config"
155
+ config_yaml_path = "#{@config_output_location}/#{@component_name}.config.yaml"
156
+ FileUtils.mkdir_p(@config_output_location) unless Dir.exist?(@config_output_location)
157
+
158
+ File.write(config_yaml_path, @component.config.to_yaml)
159
+ puts "Config for #{@component.highlander_dsl.name} written to #{config_yaml_path}"
160
+
161
+ if write_subcomponents_config
162
+ # compile sub-component templates
163
+ @sub_components.each do |sub_component|
164
+ sub_component.writeConfig write_subcomponents_config
165
+ end
166
+ end
167
+ @config_written = true
168
+ config_yaml_path
169
+ end
170
+
171
+ def processLambdas()
172
+
173
+ @component.highlander_dsl.lambda_functions_keys.each do |lfk|
174
+ resolver = LambdaResolver.new(@component,
175
+ lfk,
176
+ @workdir,
177
+ (not @silent_mode)
178
+ )
179
+ @lambda_src_paths += resolver.generateSourceArchives
180
+ resolver.mergeComponentConfig
181
+ end
182
+
183
+ @lambdas_processed = true
184
+
185
+ end
186
+
187
+ end
188
+
189
+
190
+ class LambdaResolver
191
+
192
+ def initialize(component, lambda_key, workdir, confirm_code_execution = true)
193
+ @component = component
194
+ @lambda_config = @component.config[lambda_key]
195
+ @component_dir = @component.component_dir
196
+ @workdir = workdir
197
+ @metadata = {
198
+ 'path' => {},
199
+ 'sha256' => {},
200
+ 'version' => {}
201
+ }
202
+ @confirm_code_execution = confirm_code_execution
203
+ end
204
+
205
+ def generateSourceArchives
206
+
207
+ # Clear previous packages
208
+ FileUtils.rmtree "#{@workdir}/output/lambdas"
209
+
210
+ archive_paths = []
211
+
212
+ # Cached downloads map
213
+ cached_downloads = {}
214
+ @lambda_config['functions'].each do |name, lambda_config|
215
+ # create folder
216
+ out_folder = "#{@workdir}/out/lambdas/"
217
+ timestamp = Time.now.utc.to_i.to_s
218
+ file_name = "#{name}.#{@component.name}.#{@component.version}.#{timestamp}.zip"
219
+ @metadata['path'][name] = file_name
220
+ full_destination_path = "#{out_folder}#{file_name}"
221
+ archive_paths << full_destination_path
222
+ FileUtils.mkdir_p out_folder
223
+ # clear destination if already there
224
+ FileUtils.remove full_destination_path if File.exist? full_destination_path
225
+
226
+ # download file if code remote archive
227
+ puts "Packaging AWS Lambda function #{name}...\n"
228
+ if lambda_config['code'].include? 'http'
229
+ md5 = Digest::MD5.new
230
+ md5.update lambda_config['code']
231
+ hash = md5.hexdigest
232
+ cached_location = "#{@workdir}/.cache/lambdas/#{hash}"
233
+ if cached_downloads.key? lambda_config['code']
234
+ puts "Using already downloaded archive #{lambda_config['code']}"
235
+ FileUtils.copy(cached_downloads[lambda_config['code']], full_destination_path)
236
+ elsif File.file? cached_location
237
+ puts "Using cached download - '#{cached_location}'"
238
+ FileUtils.copy(cached_location, "#{out_folder}/src.zip")
239
+ else
240
+ puts "Downloading file #{lambda_config['code']} ..."
241
+ download = open(lambda_config['code'])
242
+ IO.copy_stream(download, "#{out_folder}/src.zip")
243
+ FileUtils.mkdir_p('.cache/lambdas')
244
+ FileUtils.copy("#{out_folder}/src.zip", cached_location)
245
+ puts "Download complete, caching in #{cached_location}"
246
+ cached_downloads[lambda_config['code']] = cached_location
247
+ end
248
+ else
249
+ # zip local code
250
+ zip_options = ''
251
+ full_path = "#{@component_dir}/lambdas/#{lambda_config['code']}"
252
+
253
+ # lambda source can be either path to file or directory within that file
254
+ # optionally, lambda source code
255
+ lambda_source_dir = File.dirname(full_path)
256
+ lambda_source_file = File.basename(full_path)
257
+ lambda_source_file = '.' if Pathname.new(full_path).directory?
258
+ lambda_source_dir = full_path if Pathname.new(full_path).directory?
259
+
260
+ # executing package command can generate files. We DO NOT want this file in source directory,
261
+ # but rather in intermediate directory
262
+ tmp_source_dir = "#{@workdir}/out/lambdas/tmp/#{name}"
263
+ FileUtils.mkpath(File.dirname(tmp_source_dir))
264
+ FileUtils.copy_entry(lambda_source_dir, tmp_source_dir)
265
+ lambda_source_dir = tmp_source_dir
266
+
267
+ # Lambda function source code allows pre-processing (e.g. install code dependencies)
268
+ unless lambda_config['package_cmd'].nil?
269
+ puts "Following code will be executed to generate lambda function #{name}:\n\n#{lambda_config['package_cmd']}\n\n"
270
+
271
+ if @confirm_code_execution
272
+ exit -7 unless HighLine.agree('Proceed (y/n)?')
273
+ end
274
+
275
+ package_cmd = "cd #{lambda_source_dir} && #{lambda_config['package_cmd']}"
276
+ puts 'Processing package command...'
277
+ package_result = system(package_cmd)
278
+ unless package_result
279
+ puts "Error packaging lambda function, following command failed\n\n#{package_cmd}\n\n"
280
+ exit -4
281
+ end
282
+ end
283
+ File.delete full_destination_path if File.exist? full_destination_path
284
+ zip_generator = Highlander::Util::ZipFileGenerator.new(lambda_source_dir, full_destination_path)
285
+ zip_generator.write
286
+
287
+ end
288
+
289
+ sha256 = Digest::SHA256.file full_destination_path
290
+ sha256 = sha256.base64digest
291
+ puts "Created zip package #{full_destination_path} for lambda #{name} with digest #{sha256}"
292
+ @metadata['sha256'][name] = sha256
293
+ @metadata['version'][name] = timestamp
294
+ end
295
+
296
+ return archive_paths
297
+ end
298
+
299
+ def mergeComponentConfig
300
+ @component.config['lambda_metadata'] = @metadata
301
+ end
302
+
303
+ end
304
+
305
+ end
306
+ end
@@ -0,0 +1,25 @@
1
+
2
+ module Highlander
3
+
4
+ module Dsl
5
+ class DslBase
6
+
7
+ attr_accessor :config
8
+
9
+ def initialize(parent)
10
+ @config = parent.config unless parent.nil?
11
+ end
12
+
13
+ def method_missing(method, *args)
14
+ if @config.nil?
15
+ raise StandardError, "#{self} no config!"
16
+ end
17
+ return @config["#{method}"] unless @config["#{method}"].nil?
18
+ raise StandardError, "#{self}Unknown method or variable #{method} in Highlander template"
19
+ end
20
+
21
+ end
22
+ end
23
+
24
+ end
25
+
@@ -0,0 +1,243 @@
1
+ require_relative './highlander.helper'
2
+ require_relative './highlander.dsl.base'
3
+ require_relative './highlander.factory'
4
+
5
+ module Highlander
6
+
7
+ module Dsl
8
+
9
+ class SubcomponentParameter
10
+ attr_accessor :name, :cfndsl_value
11
+
12
+ def initialize
13
+
14
+ end
15
+
16
+ end
17
+
18
+
19
+ class Component < DslBase
20
+
21
+ attr_accessor :name,
22
+ :template,
23
+ :template_version,
24
+ :distribution_format,
25
+ :distribution_location,
26
+ :distribution_url,
27
+ :distribution_format,
28
+ :component_loaded,
29
+ :parameters,
30
+ :param_values,
31
+ :parent,
32
+ :component_config_override,
33
+ :export_config
34
+
35
+ def initialize(parent,
36
+ name,
37
+ template,
38
+ param_values,
39
+ component_sources = [],
40
+ config = {},
41
+ export_config = {},
42
+ distribution_format = 'yaml')
43
+
44
+ @parent = parent
45
+ @config = config
46
+ @export_config = export_config
47
+ @component_sources = component_sources
48
+
49
+ template_name = template
50
+ template_version = 'latest'
51
+ if template.include?('@') and not (template.start_with? 'git')
52
+ template_name = template.split('@')[0]
53
+ template_version = template.split('@')[1]
54
+ end
55
+
56
+ @template = template_name
57
+ @template_version = template_version
58
+ @name = name
59
+ @param_values = param_values
60
+
61
+ # distribution settings
62
+ @distribution_format = distribution_format
63
+ # by default components located at same location as master stack
64
+ @distribution_location = '.'
65
+ build_distribution_url
66
+
67
+ # load component
68
+ factory = Highlander::Factory::ComponentFactory.new(@component_sources)
69
+ @component_loaded = factory.findComponent(
70
+ @template,
71
+ @template_version
72
+ )
73
+ @component_loaded.config.extend @config
74
+
75
+ @parameters = []
76
+ # load_parameters
77
+ end
78
+
79
+ def version=(value)
80
+ @component_loaded.version = value
81
+ end
82
+
83
+ def distribute_bucket=(value)
84
+ @component_loaded.distribution_bucket = value
85
+ end
86
+
87
+ def distribute_prefix=(value)
88
+ @component_loaded.distribution_prefix = value
89
+ end
90
+
91
+ def distribution_format=(value)
92
+ @distribution_format = value
93
+ build_distribution_url
94
+ end
95
+
96
+ def build_distribution_url
97
+ @distribution_location = @parent.distribute_url unless @parent.distribute_url.nil?
98
+ @distribution_url = "#{@distribution_location}/#{@name}.compiled.#{@distribution_format}"
99
+ end
100
+
101
+ def load(component_config_override = {})
102
+ # check for component config on parent
103
+ parent = @parent
104
+
105
+ # Highest priority is DSL defined configuration
106
+ component_config_override.extend @config
107
+
108
+ @component_config_override = component_config_override
109
+
110
+ @component_loaded.load @component_config_override
111
+ end
112
+
113
+ # Parameters should be lazy loaded, that is late-binding should happen once
114
+ # all parameters and mappings are known
115
+ def load_parameters
116
+ component_dsl = @component_loaded.highlander_dsl
117
+ component_dsl.parameters.param_list.each do |component_param|
118
+ param = Highlander::Dsl::SubcomponentParameter.new
119
+ param.name = component_param.name
120
+ param.cfndsl_value = SubcomponentParamValueResolver.resolveValue(
121
+ @parent,
122
+ self,
123
+ component_param)
124
+ @parameters << param
125
+ end
126
+ end
127
+
128
+ end
129
+
130
+ class SubcomponentParamValueResolver
131
+ def self.resolveValue(component, sub_component, param)
132
+
133
+ puts("Resolving parameter #{component.name} -> #{sub_component.name}.#{param.name}")
134
+
135
+ # check if there are values defined on component itself
136
+ if sub_component.param_values.key?(param.name)
137
+ return Highlander::Helper.parameter_cfndsl_value(sub_component.param_values[param.name])
138
+ end
139
+
140
+ if param.class == Highlander::Dsl::StackParam
141
+ return self.resolveStackParamValue(component, sub_component, param)
142
+ elsif param.class == Highlander::Dsl::ComponentParam
143
+ return self.resolveComponentParamValue(component, sub_component, param)
144
+ elsif param.class == Highlander::Dsl::MappingParam
145
+ return self.resolveMappingParamValue(component, sub_component, param)
146
+ elsif param.class == Highlander::Dsl::OutputParam
147
+ return self.resolveOutputParamValue(component, sub_component, param)
148
+ else
149
+ raise "#{param.class} not resolvable to parameter value"
150
+ end
151
+ end
152
+
153
+ def self.resolveStackParamValue(component, sub_component, param)
154
+ param_name = param.is_global ? param.name : "#{sub_component.name}#{param.name}"
155
+ return "Ref('#{param_name}')"
156
+ end
157
+
158
+ def self.resolveComponentParamValue(component, sub_component, param)
159
+ # check component config for param value
160
+ # TODO
161
+ # check stack config for param value
162
+ # TODO
163
+ # return default value
164
+ return "'#{param.default_value}'"
165
+ end
166
+
167
+ def self.resolveMappingParamValue(component, sub_component, param)
168
+
169
+ # determine map name
170
+
171
+ provider = nil
172
+
173
+ mappings_name = param.mapName
174
+ actual_map_name = mappings_name
175
+
176
+ key_name = nil
177
+
178
+ # priority 0: stack-level parameter of map name
179
+ stack_param_mapname = component.parameters.param_list.find { |p| p.name == mappings_name }
180
+ unless stack_param_mapname.nil?
181
+ key_name = "Ref('#{mappings_name}')"
182
+ end
183
+
184
+ # priority 1 mapping provider keyName - used as lowest priority
185
+ if key_name.nil?
186
+ provider = mappings_provider(mappings_name)
187
+ if ((not provider.nil?) and (provider.respond_to?('getDefaultKey')))
188
+ key_name = provider.getDefaultKey
189
+ end
190
+ end
191
+
192
+ # priority 2: dsl defined key name
193
+ if key_name.nil?
194
+ key_name = param.mapKey
195
+ # could still be nil after this line
196
+ end
197
+
198
+ value = mapping_value(component: component,
199
+ provider_name: mappings_name,
200
+ value_name: param.mapAttribute,
201
+ key_name: key_name
202
+ )
203
+
204
+ if value.nil?
205
+ return "'#{param.default_value}'" unless param.default_value.empty?
206
+ return "''"
207
+ end
208
+
209
+ return value
210
+
211
+
212
+ return value
213
+ end
214
+
215
+ def self.resolveOutputParamValue(component, sub_component, param)
216
+ component_name = param.component
217
+ resource_name = nil
218
+ if not sub_component.export_config.nil?
219
+ if sub_component.export_config.key? component_name
220
+ resource_name = sub_component.export_config[component_name]
221
+ end
222
+ end
223
+
224
+ if resource_name.nil?
225
+ # find by component
226
+ resource = component.components.find { |c| c.name == component_name }
227
+ resource_name = resource.name unless resource.nil?
228
+ if resource_name.nil?
229
+ resource = component.components.find { |c| c.template == component_name }
230
+ resource_name = resource.name unless resource.nil?
231
+ end
232
+ end
233
+
234
+ if resource_name.nil?
235
+ raise "#{sub_component.name}.Params.#{param.name}: Failed to resolve OutputParam '#{param.name}' with source '#{component_name}'. Component not found!"
236
+ end
237
+
238
+ return "FnGetAtt('#{resource_name}','Outputs.#{param.name}')"
239
+ end
240
+ end
241
+
242
+ end
243
+ end
@@ -0,0 +1,113 @@
1
+ require_relative './highlander.dsl.base'
2
+
3
+ module Highlander
4
+
5
+ module Dsl
6
+ class Parameters < DslBase
7
+
8
+ attr_accessor :param_list
9
+
10
+ def initialize()
11
+ @param_list = []
12
+ end
13
+
14
+ def addParam(param)
15
+ existing_param = @param_list.find { |p| p.name == param.name }
16
+ if not existing_param.nil?
17
+ puts "Parameter being overwritten. Updating parameter #{param.name} with new definition..."
18
+ @param_list[@param_list.index(existing_param)] = param
19
+ else
20
+ @param_list << param
21
+ end
22
+ end
23
+
24
+ def StackParam(name, defaultValue='', isGlobal: false, noEcho: false)
25
+ param = StackParam.new(name, 'String', defaultValue)
26
+ param.is_global = isGlobal
27
+ param.config = @config
28
+ param.no_echo = noEcho
29
+ addParam param
30
+ end
31
+
32
+ def ComponentParam(name, defaultValue='')
33
+ param = ComponentParam.new(name, 'String', defaultValue)
34
+ param.config = @config
35
+ addParam param
36
+ end
37
+
38
+ def MappingParam(name, defaultValue='', &block)
39
+ param = MappingParam.new(name, 'String', defaultValue)
40
+ param.config = @config
41
+ param.instance_eval(&block)
42
+ addParam param
43
+ end
44
+
45
+ def OutputParam(component:, name:, default: '')
46
+ param = OutputParam.new(component, name, default)
47
+ param.config = @config
48
+ addParam param
49
+ end
50
+ end
51
+
52
+ class Parameter < DslBase
53
+ attr_accessor :name, :type, :default_value, :no_echo
54
+
55
+ def initialize(name, type, defaultValue, noEcho = false)
56
+ @no_echo = noEcho
57
+ @name = name
58
+ @type = type
59
+ @default_value = defaultValue
60
+ end
61
+ end
62
+
63
+ class StackParam < Parameter
64
+ attr_accessor :is_global
65
+ end
66
+
67
+ class ComponentParam < Parameter
68
+
69
+ end
70
+
71
+ class OutputParam < Parameter
72
+ attr_accessor :component
73
+
74
+ def initialize(component, name, default)
75
+ @component = component
76
+ @name = name
77
+ @default_value = default
78
+ @type = 'String'
79
+ end
80
+ end
81
+
82
+ class MappingParam < Parameter
83
+
84
+ attr_accessor :mapName, :mapKey, :mapAttribute
85
+
86
+ def method_missing(method, *args)
87
+ smethod = "#{method}"
88
+ if smethod.start_with?('Map')
89
+ puts smethod
90
+ end
91
+
92
+ super.method_missing(method)
93
+ end
94
+
95
+ def key(map_key)
96
+ @mapKey = map_key
97
+ end
98
+
99
+ def attribute(key)
100
+ @mapAttribute = key
101
+ end
102
+
103
+ def map(mapName)
104
+ @mapName = mapName
105
+ end
106
+
107
+ def mapProvider
108
+ mappings_provider(@mapName)
109
+ end
110
+
111
+ end
112
+ end
113
+ end