lono 1.1.3 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. checksums.yaml +4 -4
  2. data/.gitmodules +3 -0
  3. data/CHANGELOG.md +8 -0
  4. data/README.md +150 -39
  5. data/bin/lono +2 -2
  6. data/circle.yml +4 -0
  7. data/lib/lono.rb +16 -7
  8. data/lib/lono/cfn.rb +64 -0
  9. data/lib/lono/cfn/aws_services.rb +37 -0
  10. data/lib/lono/cfn/base.rb +144 -0
  11. data/lib/lono/cfn/create.rb +34 -0
  12. data/lib/lono/cfn/delete.rb +26 -0
  13. data/lib/lono/cfn/diff.rb +43 -0
  14. data/lib/lono/cfn/help.rb +93 -0
  15. data/lib/lono/cfn/preview.rb +133 -0
  16. data/lib/lono/cfn/update.rb +62 -0
  17. data/lib/lono/cfn/util.rb +21 -0
  18. data/lib/lono/cli.rb +19 -10
  19. data/lib/lono/command.rb +25 -0
  20. data/lib/lono/help.rb +59 -0
  21. data/lib/lono/new.rb +3 -2
  22. data/lib/lono/param.rb +20 -0
  23. data/lib/lono/param/generator.rb +90 -0
  24. data/lib/lono/param/help.rb +15 -0
  25. data/lib/lono/project_checker.rb +44 -0
  26. data/lib/lono/template.rb +22 -248
  27. data/lib/lono/template/bashify.rb +39 -0
  28. data/lib/lono/template/dsl.rb +139 -0
  29. data/lib/lono/template/help.rb +25 -0
  30. data/lib/lono/template/template.rb +251 -0
  31. data/lib/lono/version.rb +1 -1
  32. data/lib/{starter_project_yaml → starter_projects/json_project}/Gemfile +0 -1
  33. data/lib/{starter_project_json → starter_projects/json_project}/Guardfile +0 -0
  34. data/lib/{starter_project_json → starter_projects/json_project}/config/lono.rb +0 -0
  35. data/lib/{starter_project_json → starter_projects/json_project}/config/lono/api.rb +0 -0
  36. data/lib/starter_projects/json_project/params/api-web-prod.txt +20 -0
  37. data/lib/{starter_project_json → starter_projects/json_project}/templates/db.json.erb +0 -0
  38. data/lib/{starter_project_json → starter_projects/json_project}/templates/partial/host_record.json.erb +0 -0
  39. data/lib/{starter_project_json → starter_projects/json_project}/templates/partial/server.json.erb +0 -0
  40. data/lib/{starter_project_json → starter_projects/json_project}/templates/user_data/app.sh.erb +0 -0
  41. data/lib/{starter_project_json → starter_projects/json_project}/templates/user_data/db.sh.erb +0 -0
  42. data/lib/{starter_project_json → starter_projects/json_project}/templates/user_data/db2.sh.erb +0 -0
  43. data/lib/{starter_project_json → starter_projects/json_project}/templates/user_data/ruby_script.rb.erb +0 -0
  44. data/lib/{starter_project_json → starter_projects/json_project}/templates/web.json.erb +0 -0
  45. data/lib/{starter_project_json → starter_projects/yaml_project}/Gemfile +0 -1
  46. data/lib/{starter_project_yaml → starter_projects/yaml_project}/Guardfile +0 -0
  47. data/lib/{starter_project_yaml → starter_projects/yaml_project}/config/lono.rb +0 -0
  48. data/lib/{starter_project_yaml → starter_projects/yaml_project}/config/lono/api.rb +0 -0
  49. data/lib/starter_projects/yaml_project/params/api-web-prod.txt +20 -0
  50. data/lib/{starter_project_yaml → starter_projects/yaml_project}/templates/db.yml.erb +0 -0
  51. data/lib/{starter_project_yaml → starter_projects/yaml_project}/templates/partial/host_record.yml.erb +0 -0
  52. data/lib/{starter_project_yaml → starter_projects/yaml_project}/templates/partial/server.yml.erb +0 -0
  53. data/lib/{starter_project_yaml → starter_projects/yaml_project}/templates/partial/user_data/bootstrap.sh.erb +0 -0
  54. data/lib/{starter_project_yaml → starter_projects/yaml_project}/templates/web.yml.erb +0 -0
  55. data/lono.gemspec +15 -10
  56. data/spec/fixtures/my_project/config/lono.rb +1 -0
  57. data/spec/fixtures/my_project/params/my-stack.txt +3 -0
  58. data/spec/fixtures/my_project/templates/.gitkeep +0 -0
  59. data/spec/fixtures/my_project/templates/my-stack.yml.erb +0 -0
  60. data/spec/lib/lono/cfn_spec.rb +35 -0
  61. data/spec/lib/lono/new_spec.rb +3 -3
  62. data/spec/lib/lono/param_spec.rb +15 -0
  63. data/spec/lib/lono/{dsl_spec.rb → template/dsl_spec.rb} +9 -9
  64. data/spec/lib/lono/template/template_spec.rb +104 -0
  65. data/spec/lib/lono/template_spec.rb +22 -37
  66. data/spec/lib/lono_spec.rb +6 -83
  67. data/vendor/plissken/Gemfile +14 -0
  68. data/vendor/plissken/LICENSE.txt +20 -0
  69. data/vendor/plissken/README.md +46 -0
  70. data/vendor/plissken/Rakefile +56 -0
  71. data/vendor/plissken/VERSION +1 -0
  72. data/vendor/plissken/lib/plissken.rb +1 -0
  73. data/vendor/plissken/lib/plissken/ext/hash/to_snake_keys.rb +45 -0
  74. data/vendor/plissken/plissken.gemspec +61 -0
  75. data/vendor/plissken/spec/lib/to_snake_keys_spec.rb +177 -0
  76. data/vendor/plissken/spec/spec_helper.rb +90 -0
  77. data/vendor/plissken/test/helper.rb +20 -0
  78. data/vendor/plissken/test/plissken/ext/hash/to_snake_keys_test.rb +184 -0
  79. data/vendor/plissken/test/test_plissken.rb +2 -0
  80. metadata +115 -39
  81. data/lib/lono/bashify.rb +0 -41
  82. data/lib/lono/cli/help.rb +0 -37
  83. data/lib/lono/dsl.rb +0 -132
@@ -0,0 +1,39 @@
1
+ require 'open-uri'
2
+
3
+ class Lono::Template::Bashify
4
+ def initialize(options={})
5
+ @options = options
6
+ @path = options[:path]
7
+ end
8
+
9
+ def user_data_paths(data,path="")
10
+ paths = []
11
+ paths << path
12
+ data.each do |key,value|
13
+ if value.is_a?(Hash)
14
+ paths += user_data_paths(value,"#{path}/#{key}")
15
+ else
16
+ paths += ["#{path}/#{key}"]
17
+ end
18
+ end
19
+ paths.select {|p| p =~ /UserData/ && p =~ /Fn::Join/ }
20
+ end
21
+
22
+ def run
23
+ raw = open(@path).read
24
+ json = JSON.load(raw)
25
+ paths = user_data_paths(json)
26
+ if paths.empty?
27
+ puts "No UserData script found"
28
+ return
29
+ end
30
+ paths.each do |path|
31
+ puts "UserData script for #{path}:"
32
+ key = path.sub('/','').split("/").map {|x| "['#{x}']"}.join('')
33
+ user_data = eval("json#{key}")
34
+ delimiter = user_data[0]
35
+ script = user_data[1]
36
+ puts script.join(delimiter)
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,139 @@
1
+ class Lono::Template::DSL
2
+ def initialize(options={})
3
+ @options = options
4
+ @project_root = @options[:project_root] || '.'
5
+ @path = "#{@project_root}/config/lono.rb"
6
+ Lono::ProjectChecker.check(@project_root)
7
+ @templates = []
8
+ @results = {}
9
+ @detected_format = nil
10
+ end
11
+
12
+ def run(options={})
13
+ evaluate_templates
14
+ build_templates
15
+ write_output
16
+ end
17
+
18
+ def evaluate_templates
19
+ instance_eval(File.read(@path), @path)
20
+ load_subfolder
21
+ @detected_format = detect_format
22
+ end
23
+
24
+ # Detects the format of the templates. Simply checks the extension of all the
25
+ # templates files.
26
+ # All the templates must be of the same format, either all json or all yaml.
27
+ def detect_format
28
+ # @templates contains Array of Hashes. Example:
29
+ # [{name: ""blog-web-prod.json", block: ...},
30
+ # {name: ""api-web-prod.json", block: ...}]
31
+ formats = @templates.map{ |t| File.extname(t[:name]) }.uniq
32
+ if formats.size > 1
33
+ puts "ERROR: Detected multiple formats: #{formats.join(", ")}".colorize(:red)
34
+ puts "All the source values in the template blocks in the config folder must have the same format extension."
35
+ exit 1
36
+ else
37
+ found_format = formats.first
38
+ if found_format
39
+ detected_format = found_format.sub(/^\./,'')
40
+ detected_format = "yaml" if detected_format == "yml"
41
+ else # empty templates, no templates defined yet
42
+ detected_format = "yaml" # defaults to yaml
43
+ end
44
+ end
45
+ detected_format
46
+ end
47
+
48
+ # load any templates defined in project/config/lono/*
49
+ def load_subfolder
50
+ Dir.glob("#{File.dirname(@path)}/lono/**/*").select{ |e| File.file? e }.each do |path|
51
+ instance_eval(File.read(path), path)
52
+ end
53
+ end
54
+
55
+ def template(name, &block)
56
+ @templates << {name: name, block: block}
57
+ end
58
+
59
+ def build_templates
60
+ @templates.each do |t|
61
+ @results[t[:name]] = Lono::Template::Template.new(t[:name], t[:block], @options).build
62
+ end
63
+ end
64
+
65
+ def write_output
66
+ output_path = "#{@project_root}/output"
67
+ FileUtils.rm_rf(output_path) if @options[:clean]
68
+ FileUtils.mkdir(output_path) unless File.exist?(output_path)
69
+ puts "Generating CloudFormation templates:" unless @options[:quiet]
70
+ @results.each do |name,text|
71
+ path = "#{output_path}/#{name}".sub(/^\.\//,'')
72
+ puts " #{path}" unless @options[:quiet]
73
+ ensure_parent_dir(path)
74
+ validate(text, path)
75
+ File.open(path, 'w') do |f|
76
+ f.write(output_format(text))
77
+ end
78
+ end
79
+ end
80
+
81
+ # TODO: set @detected_format upon DSL.new
82
+ def validate(text, path)
83
+ if @detected_format == "json"
84
+ validate_json(text, path)
85
+ else
86
+ validate_yaml(text, path)
87
+ end
88
+ end
89
+
90
+ def validate_yaml(yaml, path)
91
+ begin
92
+ YAML.load(yaml)
93
+ rescue Psych::SyntaxError => e
94
+ puts "Invalid yaml. Output written to #{path} for debugging".colorize(:red)
95
+ puts "ERROR: #{e.message}".colorize(:red)
96
+ File.open(path, 'w') {|f| f.write(yaml) }
97
+ exit 1
98
+ end
99
+ end
100
+
101
+ def validate_json(json, path)
102
+ begin
103
+ JSON.parse(json)
104
+ rescue JSON::ParserError => e
105
+ puts "Invalid json. Output written to #{path} for debugging".colorize(:red)
106
+ puts "ERROR: #{e.message}".colorize(:red)
107
+ File.open(path, 'w') {|f| f.write(json) }
108
+ exit 1
109
+ end
110
+ end
111
+
112
+ def output_format(text)
113
+ @options[:pretty] ? prettify(text) : text
114
+ end
115
+
116
+ # Input text is either yaml or json.
117
+ # Do not prettify yaml format because it removes the !Ref like CloudFormation notation
118
+ def prettify(text)
119
+ @detected_format == "json" ? JSON.pretty_generate(JSON.parse(text)) : yaml_format(text)
120
+ end
121
+
122
+ def yaml_format(text)
123
+ comment =<<~EOS
124
+ # This file was generated with lono. Do not edit directly, the changes will be lost.
125
+ # More info: https://github.com/tongueroo/lono
126
+ EOS
127
+ "#{comment}#{remove_blank_lines(text)}"
128
+ end
129
+
130
+ # ERB templates leaves blank lines around, remove those lines
131
+ def remove_blank_lines(text)
132
+ text.split("\n").reject { |l| l.strip == '' }.join("\n") + "\n"
133
+ end
134
+
135
+ def ensure_parent_dir(path)
136
+ dir = File.dirname(path)
137
+ FileUtils.mkdir_p(dir) unless File.exist?(dir)
138
+ end
139
+ end
@@ -0,0 +1,25 @@
1
+ module Lono::Template::Help
2
+ def generate
3
+ <<-EOL
4
+ Examples:
5
+
6
+ $ lono template generate
7
+
8
+ $ lono template g -c # shortcut
9
+
10
+ Builds the CloudFormation templates files based on lono project and writes them to the output folder on the filesystem.
11
+ EOL
12
+ end
13
+
14
+ def bashify
15
+ <<-EOL
16
+ Examples:
17
+
18
+ $ lono template bashify /path/to/cloudformation-template.json
19
+
20
+ $ lono template bashify https://s3.amazonaws.com/cloudformation-templates-us-east-1/EC2WebSiteSample.template
21
+ EOL
22
+ end
23
+
24
+ extend self
25
+ end
@@ -0,0 +1,251 @@
1
+ require 'erb'
2
+ require 'json'
3
+ require 'base64'
4
+
5
+ class Lono::Template::Template
6
+ include ERB::Util
7
+
8
+ attr_reader :name
9
+ def initialize(name, block, options={})
10
+ @name = name
11
+ @block = block
12
+ @options = options
13
+ end
14
+
15
+ def build
16
+ instance_eval(&@block)
17
+ template = IO.read(@source)
18
+ erb_result(@source, template)
19
+ end
20
+
21
+ def source(path)
22
+ @source = path[0..0] == '/' ? path : "#{@options[:project_root]}/templates/#{path}"
23
+ end
24
+
25
+ def variables(vars={})
26
+ vars.each do |var,value|
27
+ instance_variable_set("@#{var}", value)
28
+ end
29
+ end
30
+
31
+ def partial(path,vars={}, options={})
32
+ path = "#{@options[:project_root]}/templates/partial/#{path}"
33
+ template = IO.read(path)
34
+ variables(vars)
35
+ result = erb_result(path, template)
36
+ result = indent(result, options[:indent]) if options[:indent]
37
+ result
38
+ end
39
+
40
+ # add indentation
41
+ def indent(result, indentation_amount)
42
+ result.split("\n").map do |line|
43
+ " " * indentation_amount + line
44
+ end.join("\n")
45
+ end
46
+
47
+ def erb_result(path, template)
48
+ begin
49
+ ERB.new(template, nil, "-").result(binding)
50
+ rescue Exception => e
51
+ puts e
52
+
53
+ # how to know where ERB stopped? - https://www.ruby-forum.com/topic/182051
54
+ # syntax errors have the (erb):xxx info in e.message
55
+ # undefined variables have (erb):xxx info in e.backtrac
56
+ error_info = e.message.split("\n").grep(/\(erb\)/)[0]
57
+ error_info ||= e.backtrace.grep(/\(erb\)/)[0]
58
+ raise unless error_info # unable to find the (erb):xxx: error line
59
+ line = error_info.split(':')[1].to_i
60
+ puts "Error evaluating ERB template on line #{line.to_s.colorize(:red)} of: #{path.sub(/^\.\//, '')}"
61
+
62
+ template_lines = template.split("\n")
63
+ context = 5 # lines of context
64
+ top, bottom = [line-context-1, 0].max, line+context-1
65
+ spacing = template_lines.size.to_s.size
66
+ template_lines[top..bottom].each_with_index do |line_content, index|
67
+ line_number = top+index+1
68
+ if line_number == line
69
+ printf("%#{spacing}d %s\n".colorize(:red), line_number, line_content)
70
+ else
71
+ printf("%#{spacing}d %s\n", line_number, line_content)
72
+ end
73
+ end
74
+ exit 1 unless ENV['TEST']
75
+ end
76
+ end
77
+
78
+ def user_data(path, vars={})
79
+ path = "#{@options[:project_root]}/templates/user_data/#{path}"
80
+ template = IO.read(path)
81
+ variables(vars)
82
+ result = erb_result(path, template)
83
+ output = []
84
+ result.split("\n").each do |line|
85
+ output += transform(line)
86
+ end
87
+ json = output.to_json
88
+ json[0] = '' # remove first char: [
89
+ json.chop! # remove last char: ]
90
+ end
91
+
92
+ def ref(name)
93
+ %Q|{"Ref"=>"#{name}"}|
94
+ end
95
+
96
+ def find_in_map(*args)
97
+ %Q|{"Fn::FindInMap" => [ #{transform_array(args)} ]}|
98
+ end
99
+
100
+ def base64(value)
101
+ %Q|{"Fn::Base64"=>"#{value}"}|
102
+ end
103
+
104
+ def get_att(*args)
105
+ %Q|{"Fn::GetAtt" => [ #{transform_array(args)} ]}|
106
+ end
107
+
108
+ def get_azs(region="AWS::Region")
109
+ %Q|{"Fn::GetAZs"=>"#{region}"}|
110
+ end
111
+
112
+ def join(delimiter, values)
113
+ %Q|{"Fn::Join" => ["#{delimiter}", [ #{transform_array(values)} ]]}|
114
+ end
115
+
116
+ def select(index, list)
117
+ %Q|{"Fn::Select" => ["#{index}", [ #{transform_array(list)} ]]}|
118
+ end
119
+
120
+ def transform_array(arr)
121
+ arr.map! {|x| x =~ /=>/ ? x : x.inspect }
122
+ arr.join(',')
123
+ end
124
+
125
+ # transform each line of bash script to array with cloudformation template objects
126
+ def transform(data)
127
+ data = evaluate(data)
128
+ if data[-1].is_a?(String)
129
+ data[0..-2] + ["#{data[-1]}\n"]
130
+ else
131
+ data + ["\n"]
132
+ end
133
+ end
134
+
135
+ # Input:
136
+ # String
137
+ # Output:
138
+ # Array of parse positions
139
+ #
140
+ # The positions of tokens taking into account when brackets start and close,
141
+ # handles nested brackets.
142
+ def bracket_positions(line)
143
+ positions,pair,count = [],[],0
144
+
145
+ line.split('').each_with_index do |char,i|
146
+ pair << i if pair.empty?
147
+
148
+ first_pair_char = line[pair[0]]
149
+ if first_pair_char == '{' # object logic
150
+ if char == '{'
151
+ count += 1
152
+ end
153
+
154
+ if char == '}'
155
+ count -= 1
156
+ if count == 0
157
+ pair << i
158
+ positions << pair
159
+ pair = []
160
+ end
161
+ end
162
+ else # string logic
163
+ lookahead = line[i+1]
164
+ if lookahead == '{'
165
+ pair << i
166
+ positions << pair
167
+ pair = []
168
+ end
169
+ end
170
+ end # end of loop
171
+
172
+ # for string logic when lookahead does not contain a object token
173
+ # need to clear out what's left to match the final pair
174
+ if !pair.empty?
175
+ pair << line.size - 1
176
+ positions << pair
177
+ end
178
+
179
+ positions
180
+ end
181
+
182
+ # Input:
183
+ # Array - bracket_positions
184
+ # Ouput:
185
+ # Array - positions that can be use to determine what to parse
186
+ def parse_positions(line)
187
+ positions = bracket_positions(line)
188
+ positions.flatten
189
+ end
190
+
191
+ # Input
192
+ # String line of code to decompose into chunks, some can be transformed into objects
193
+ # Output
194
+ # Array of strings, some can be transformed into objects
195
+ #
196
+ # Example:
197
+ # line = 'a{b}c{d{d}d}e' # nested brackets
198
+ # template.decompose(line).should == ['a','{b}','c','{d{d}d}','e']
199
+ def decompose(line)
200
+ positions = parse_positions(line)
201
+ return [line] if positions.empty?
202
+
203
+ result = []
204
+ str = ''
205
+ until positions.empty?
206
+ left = positions.shift
207
+ right = positions.shift
208
+ token = line[left..right]
209
+ # if cfn object, add to the result set but after clearing out
210
+ # the temp str that is being built up when the token is just a string
211
+ if cfn_object?(token)
212
+ unless str.empty? # first token might be a object
213
+ result << str
214
+ str = ''
215
+ end
216
+ result << token
217
+ else
218
+ str << token # keeps building up the string
219
+ end
220
+ end
221
+
222
+ # at the of the loop there's a leftover string, unless the last token
223
+ # is an object
224
+ result << str unless str.empty?
225
+
226
+ result
227
+ end
228
+
229
+ def cfn_object?(s)
230
+ exact = %w[Ref]
231
+ pattern = %w[Fn::]
232
+ exact_match = !!exact.detect {|word| s.include?(word)}
233
+ pattern_match = !!pattern.detect {|p| s =~ Regexp.new(p)}
234
+ (exact_match || pattern_match) && s =~ /^{/ && s =~ /=>/
235
+ end
236
+
237
+ def recompose(decomposition)
238
+ decomposition.map { |s| cfn_object?(s) ? eval(s) : s }
239
+ end
240
+
241
+ def evaluate(line)
242
+ recompose(decompose(line))
243
+ end
244
+
245
+ # For simple just parameters files that can also be generated with lono, the CFN
246
+ # Fn::Base64 function is not available and as lono is not being used in the context
247
+ # of CloudFormation. So this can be used in it's place.
248
+ def encode_base64(text)
249
+ Base64.strict_encode64(text).strip
250
+ end
251
+ end