lono 7.3.2 → 7.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +4 -0
  3. data/lib/lono.rb +1 -0
  4. data/lib/lono/app_file/build.rb +22 -34
  5. data/lib/lono/app_file/build/lambda_layer/ruby_packager.rb +2 -33
  6. data/lib/lono/app_file/registry/item.rb +18 -45
  7. data/lib/lono/app_file/upload.rb +1 -25
  8. data/lib/lono/cfn/base.rb +8 -2
  9. data/lib/lono/cfn/create.rb +3 -2
  10. data/lib/lono/cfn/opts.rb +8 -0
  11. data/lib/lono/cfn/preview/changeset.rb +1 -0
  12. data/lib/lono/cfn/update.rb +1 -0
  13. data/lib/lono/configset.rb +1 -1
  14. data/lib/lono/configset/combiner.rb +40 -13
  15. data/lib/lono/configset/generator.rb +6 -1
  16. data/lib/lono/configset/list.rb +1 -1
  17. data/lib/lono/configset/register/dsl.rb +3 -3
  18. data/lib/lono/configset/resolver.rb +9 -5
  19. data/lib/lono/configset/s3_file/build.rb +33 -0
  20. data/lib/lono/configset/s3_file/item.rb +30 -0
  21. data/lib/lono/configset/s3_file/registry.rb +12 -0
  22. data/lib/lono/configset/s3_file/upload.rb +12 -0
  23. data/lib/lono/configset/strategy/base.rb +21 -1
  24. data/lib/lono/configset/strategy/dsl.rb +9 -6
  25. data/lib/lono/configset/strategy/erb.rb +6 -1
  26. data/lib/lono/configset/strategy/helpers/dsl.rb +8 -0
  27. data/lib/lono/configset/strategy/helpers/dsl/auth.rb +37 -0
  28. data/lib/lono/configset/strategy/{dsl/helpers → helpers/dsl}/core.rb +6 -1
  29. data/lib/lono/configset/strategy/helpers/dsl/package.rb +40 -0
  30. data/lib/lono/configset/strategy/helpers/dsl/syntax.rb +74 -0
  31. data/lib/lono/configset/strategy/{erb/helpers.rb → helpers/erb.rb} +2 -2
  32. data/lib/lono/extension/list.rb +2 -1
  33. data/lib/lono/extensions/loader.rb +2 -1
  34. data/lib/lono/file_uploader.rb +3 -109
  35. data/lib/lono/finder/base.rb +1 -1
  36. data/lib/lono/finder/blueprint/configset.rb +3 -3
  37. data/lib/lono/generate.rb +11 -1
  38. data/lib/lono/jade/registry.rb +10 -1
  39. data/lib/lono/jadespec.rb +4 -2
  40. data/lib/lono/registration/temp.rb +4 -2
  41. data/lib/lono/s3/uploader.rb +94 -0
  42. data/lib/lono/seed/base.rb +2 -1
  43. data/lib/lono/template/configset_injector.rb +6 -4
  44. data/lib/lono/template/generator.rb +1 -9
  45. data/lib/lono/template/helper.rb +2 -3
  46. data/lib/lono/template/post_processor.rb +9 -1
  47. data/lib/lono/template/strategy/dsl/builder/helpers.rb +1 -0
  48. data/lib/lono/template/strategy/dsl/builder/helpers/ec2_helper.rb +27 -0
  49. data/lib/lono/template/strategy/dsl/builder/helpers/s3_helper.rb +1 -0
  50. data/lib/lono/template/upload.rb +4 -98
  51. data/lib/lono/utils/item/file_methods.rb +29 -0
  52. data/lib/lono/utils/item/zip.rb +42 -0
  53. data/lib/lono/utils/rsync.rb +36 -0
  54. data/lib/lono/version.rb +1 -1
  55. data/lib/templates/blueprint/.gitignore +4 -7
  56. data/lib/templates/configset/%configset_name%.gemspec.tt +1 -1
  57. data/lib/templates/configset/.gitignore +2 -4
  58. data/lib/templates/extension/%extension_name%.gemspec.tt +2 -1
  59. data/lib/templates/extension/.gitignore +11 -17
  60. data/lib/templates/extension/lib/%extension_name%.rb.tt +3 -0
  61. data/lib/templates/extension/lib/%extension_name%/autoloader.rb.tt +23 -0
  62. data/lono.gemspec +1 -0
  63. metadata +32 -6
  64. data/lib/lono/configset/strategy/dsl/helpers.rb +0 -5
  65. data/lib/lono/configset/strategy/dsl/syntax.rb +0 -32
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: eafe64d8f51c8ca12fdbca5fb6cba0643389a834bca76d1031622e643322d9c2
4
- data.tar.gz: 746097bef6f91a86d8079294b9119dd3d6c6c79f707203fd716c6239a72f9198
3
+ metadata.gz: c0b9294948bbd81eea5198852bffe28c7682aaf9a73cba1273f9fa0b5f2fd7d7
4
+ data.tar.gz: 948e6cb7dd9ee9ecf5ab02c85f37687339598535bb34deff83dbf02ddc511cf4
5
5
  SHA512:
6
- metadata.gz: e98615585a826093a393b8c082b4f879ada01c1f12df4e516c56e2aa662cd04b149bc113a1ea24213079799d5769bfbd7d5524f25d377703b0d58d598f3e9df6
7
- data.tar.gz: 8c94729c489105f755648dde84cfa154143823208d4c52683b2f6d1e21014e618ae599ea559113292f15cf44c15dc09c99da659705015c210bec1a5647a2705a
6
+ metadata.gz: 7d14a3286bf4b0aba8e2a7c1129abc222b4379dd4c5f3214bcb906c08a0e078ad4455bdd68fcf7e1f774a382dd19e7e0704dacbaf0236517b5c1dce23927af38
7
+ data.tar.gz: f3a65caf45ca739360cae97be4f16e5eb45c18030eb537e1cbdccbfefa5484e52368ba656c057b75818e1e26bbdb093f173d72ffbab48d4d317f1b4e5dbfe594
@@ -3,6 +3,10 @@
3
3
  All notable changes to this project will be documented in this file.
4
4
  This project *tries* to adhere to [Semantic Versioning](http://semver.org/), even before v1.0.
5
5
 
6
+ ## [7.4.0]
7
+ - #57 Big improvements generally and configsets
8
+ - #58 Docs: Configsets and new DSL Helpers
9
+
6
10
  ## [7.3.2]
7
11
  - #56 build lambda layer fix: allow Gemfile.lock to be generated if not provided
8
12
 
@@ -3,6 +3,7 @@ $stdout.sync = true unless ENV["LONO_STDOUT_SYNC"] == "0"
3
3
  require "active_support/core_ext/class"
4
4
  require 'active_support/core_ext/hash'
5
5
  require 'active_support/core_ext/string'
6
+ require 'aws_data'
6
7
  require 'cfn_camelizer'
7
8
  require 'fileutils'
8
9
  require 'json'
@@ -2,6 +2,8 @@ require "thor"
2
2
 
3
3
  module Lono::AppFile
4
4
  class Build < Base
5
+ include Lono::Utils::Item::Zip
6
+
5
7
  def initialize_variables
6
8
  @output_files_path = "#{Lono.config.output_path}/#{@blueprint}/files"
7
9
  end
@@ -14,11 +16,28 @@ module Lono::AppFile
14
16
 
15
17
  def build_all
16
18
  clean_output
19
+ validate_files!
17
20
  copy_to_output
18
21
  build_layers
19
22
  compress_output
20
23
  end
21
24
 
25
+ def validate_files!
26
+ items = Registry.items + Registry.layers
27
+ missing = items.select do |item|
28
+ !File.exist?(item.src_path)
29
+ end
30
+ missing_paths = missing.map { |item| item.src_path }.uniq
31
+ unless missing_paths.empty?
32
+ puts "ERROR: These app/files are missing were used by the s3_key method but are missing".color(:red)
33
+ missing_paths.each do |path|
34
+ puts " #{path}"
35
+ end
36
+ puts "Please double check that they exist."
37
+ exit 1
38
+ end
39
+ end
40
+
22
41
  def build_layers
23
42
  layer_items = Registry.layers
24
43
  layer_items.each do |item|
@@ -29,12 +48,10 @@ module Lono::AppFile
29
48
  def compress_output
30
49
  Registry.items.each do |item|
31
50
  # type can be lambda_layer or file
32
- if item.type == "lambda_layer" || item.directory?
33
- zip_directory(item)
34
- elsif item.file?
35
- zip_file(item)
51
+ if item.type == "lambda_layer" || item.exist?
52
+ zip(item)
36
53
  else
37
- puts "WARN: #{item.path} does not exist. Double check that the path is correct in the s3_key call.".color(:yellow)
54
+ puts "WARN: #{item.src_path} does not exist. Double check that the path is correct in the s3_key call.".color(:yellow)
38
55
  end
39
56
  end
40
57
  end
@@ -50,35 +67,6 @@ module Lono::AppFile
50
67
  end
51
68
  memoize :context
52
69
 
53
- def zip_file(item)
54
- path = item.path
55
- zip_file = item.zip_file_name
56
-
57
- puts "Zipping file and generating md5 named file from: #{path}"
58
- command = "cd #{File.dirname(path)} && zip -q #{zip_file} #{File.basename(path)}" # create zipfile at same level of file
59
- zip(command)
60
- end
61
-
62
- def zip_directory(item)
63
- path = item.path
64
- zip_file = item.zip_file_name
65
-
66
- puts "Zipping folder and generating md5 named file from: #{path}"
67
- command = "cd #{path} && zip --symlinks -rq #{zip_file} ." # create zipfile witih directory
68
- zip(command)
69
- FileUtils.mv("#{path}/#{zip_file}", "#{File.dirname(path)}/#{zip_file}") # move zip back to the parent directory
70
- end
71
-
72
- def zip(command)
73
- # puts "=> #{command}".color(:green) # uncomment to debug
74
- `#{command}`
75
- unless $?.success?
76
- puts "ERROR: Fail to run #{command}".color(:red)
77
- puts "Maybe zip is not installed or path is incorrect?"
78
- exit 1
79
- end
80
- end
81
-
82
70
  def clean_output
83
71
  FileUtils.rm_rf(@output_files_path)
84
72
  end
@@ -1,8 +1,8 @@
1
- require 'shellwords'
2
-
3
1
  class Lono::AppFile::Build::LambdaLayer
4
2
  # Based on jets
5
3
  class RubyPackager
4
+ include Lono::Utils::Rsync
5
+
6
6
  def initialize(blueprint, registry_item)
7
7
  @blueprint, @registry_item = blueprint, registry_item
8
8
 
@@ -157,36 +157,5 @@ EOL
157
157
  gemfile_path = "#{@app_root}/Gemfile"
158
158
  File.exist?(gemfile_path)
159
159
  end
160
-
161
- def sh(command)
162
- puts "=> #{command}"
163
- out = `#{command}`
164
- puts out if ENV['LONO_DEBUG_LAMBDA_LAYER']
165
- success = $?.success?
166
- raise("ERROR: running command #{command}").color(:red) unless success
167
- success
168
- end
169
-
170
- def rsync(src, dest)
171
- # Using FileUtils.cp_r doesnt work if there are special files like socket files in the src dir.
172
- # Instead of using this hack https://bugs.ruby-lang.org/issues/10104
173
- # Using rsync to perform the copy.
174
- src.chop! if src.ends_with?('/')
175
- dest.chop! if dest.ends_with?('/')
176
- check_rsync_installed!
177
- # Ensures required trailing slashes
178
- FileUtils.mkdir_p(File.dirname(dest))
179
- sh "rsync -a --links --no-specials --no-devices #{Shellwords.escape(src)}/ #{Shellwords.escape(dest)}/"
180
- end
181
-
182
- @@rsync_installed = false
183
- def check_rsync_installed!
184
- return if @@rsync_installed # only check once
185
- if system "type rsync > /dev/null 2>&1"
186
- @@rsync_installed = true
187
- else
188
- raise "ERROR: Rsync is required. Rsync does not seem to be installed.".color(:red)
189
- end
190
- end
191
160
  end
192
161
  end
@@ -1,50 +1,23 @@
1
- module Lono::AppFile
2
- class Registry
3
- # Holds metadata about the item in the regsitry.
4
- class Item
5
- attr_reader :name, :options, :type
6
- def initialize(name, blueprint, options={})
7
- @name, @blueprint, @options = name, blueprint, options
8
- @type = options[:type] || "file"
9
- end
10
-
11
- def path
12
- if @type == "file"
13
- "#{Lono.root}/output/#{@blueprint}/files/#{@name}"
14
- else
15
- "#{Lono.root}/output/#{@blueprint}/lambda_layers/#{@name}/opt"
16
- end
17
- end
18
-
19
- def directory?
20
- File.directory?(path)
21
- end
22
-
23
- def file?
24
- File.file?(path)
25
- end
26
-
27
- def s3_path
28
- file_path = zip_file_path.sub(%r{.*/output/[\w_-]+/(files|lambda_layers)/}, '') # dont use basename. there might be subfolders
29
- "#{s3_prefix}/#{file_path}"
30
- end
31
-
32
- # full path
33
- def zip_file_path
34
- "#{File.dirname(path)}/#{zip_file_name}"
35
- end
36
-
37
- def zip_file_name
38
- "#{File.basename(path)}-#{@type}-#{Lono::Md5.sum(path)}.zip"
39
- end
1
+ class Lono::AppFile::Registry
2
+ # Holds metadata about the item in the regsitry.
3
+ class Item
4
+ include Lono::Utils::Item::FileMethods
5
+
6
+ attr_reader :name, :options, :type
7
+ def initialize(name, blueprint, options={})
8
+ @name, @blueprint, @options = name, blueprint, options
9
+ @type = options[:type] || "file"
10
+ end
40
11
 
41
- private
42
- def md5_path
43
- Lono::Md5.name(path)
44
- end
12
+ def src_path
13
+ "#{Lono.blueprint_root}/app/files/#{@name}"
14
+ end
45
15
 
46
- def s3_prefix
47
- "#{Lono.env}/#{@blueprint}/files" # development/ecs-asg/files
16
+ def output_path
17
+ if @type == "file"
18
+ "#{Lono.root}/output/#{@blueprint}/files/#{@name}"
19
+ else
20
+ "#{Lono.root}/output/#{@blueprint}/lambda_layers/#{@name}/opt"
48
21
  end
49
22
  end
50
23
  end
@@ -1,36 +1,12 @@
1
1
  module Lono::AppFile
2
2
  class Upload < Base
3
- include Lono::AwsServices
4
- extend Memoist
5
-
6
3
  def upload
7
4
  return unless Registry.items.size > 0
8
5
  puts "Uploading app/files..."
9
6
 
10
7
  Registry.items.each do |item|
11
- s3_upload(item)
8
+ Lono::S3::Uploader.new(item.zip_file_path).upload
12
9
  end
13
10
  end
14
-
15
- # TODO: check md5sum and only upload if it changes
16
- def s3_upload(item)
17
- filepath = item.zip_file_path
18
- s3_key = item.s3_path
19
- s3_path = "s3://#{s3_bucket}/#{s3_key}"
20
- message = "Uploading: #{filepath} to #{s3_path}".color(:green)
21
- message = "NOOP: #{message}" if @options[:noop]
22
- puts message
23
- return if @options[:noop]
24
-
25
- s3.put_object(
26
- body: IO.read(filepath),
27
- bucket: s3_bucket,
28
- key: s3_key,
29
- )
30
- end
31
-
32
- def s3_bucket
33
- Lono::S3::Bucket.name
34
- end
35
11
  end
36
12
  end
@@ -167,6 +167,13 @@ class Lono::Cfn
167
167
  end
168
168
  end
169
169
 
170
+ def notification_arns
171
+ @setting ||= Lono::Setting.new
172
+ settings = @setting.data
173
+ arns = @options["notification_arns"] || settings["notification_arns"]
174
+ arns == [''] ? [] : arns # allow removing the notification_arns setting
175
+ end
176
+
170
177
  def show_options(options, meth=nil)
171
178
  options = options.clone.compact
172
179
  if options[:template_body] # continue_update_rollback
@@ -185,9 +192,8 @@ class Lono::Cfn
185
192
  #
186
193
  # Reference: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/cloudformation-limits.html
187
194
  def set_template_url!(options)
188
- upload = Lono::Template::Upload.new(@options)
189
195
  url_path = template_path.sub("#{Lono.root}/",'')
190
- url = upload.s3_presigned_url(url_path)
196
+ url = Lono::S3::Uploader.new(url_path).presigned_url
191
197
  url.gsub!(/\.yml.*/, ".yml") # Interesting dont need presign query string. For stack sets it actually breaks it. So removing.
192
198
  options[:template_url] = url
193
199
  options
@@ -24,11 +24,12 @@ class Lono::Cfn
24
24
  end
25
25
 
26
26
  options = {
27
- stack_name: @stack,
28
- parameters: parameters,
29
27
  capabilities: capabilities, # ["CAPABILITY_IAM", "CAPABILITY_NAMED_IAM"]
30
28
  disable_rollback: !@options[:rollback],
29
+ parameters: parameters,
30
+ stack_name: @stack,
31
31
  }
32
+ options[:notification_arns] = notification_arns if notification_arns
32
33
  options[:tags] = tags unless tags.empty?
33
34
  set_template_url!(options)
34
35
 
@@ -1,11 +1,13 @@
1
1
  class Lono::Cfn
2
2
  class Opts < Lono::Opts
3
3
  def create
4
+ notification_arns_option
4
5
  base_options
5
6
  wait_options
6
7
  end
7
8
 
8
9
  def update
10
+ notification_arns_option
9
11
  base_options
10
12
  update_options
11
13
  wait_options
@@ -76,5 +78,11 @@ class Lono::Cfn
76
78
  option :param_preview, type: :boolean, default: true, desc: "Show parameter diff preview."
77
79
  end
78
80
  end
81
+
82
+ def notification_arns_option
83
+ with_cli_scope do
84
+ option :notification_arns, type: :array, desc: "The Simple Notification Service (SNS) topic ARNs to publish stack related events."
85
+ end
86
+ end
79
87
  end
80
88
  end
@@ -31,6 +31,7 @@ module Lono::Cfn::Preview
31
31
  stack_name: @stack,
32
32
  parameters: parameters,
33
33
  }
34
+ options[:notification_arns] = notification_arns if notification_arns
34
35
  options[:tags] = tags unless tags.empty?
35
36
  set_template_url!(options)
36
37
  show_options(options, "cfn.create_change_set")
@@ -46,6 +46,7 @@ class Lono::Cfn
46
46
  capabilities: capabilities, # ["CAPABILITY_IAM", "CAPABILITY_NAMED_IAM"]
47
47
  disable_rollback: !@options[:rollback],
48
48
  }
49
+ options[:notification_arns] = notification_arns if notification_arns
49
50
  options[:tags] = tags unless tags.empty?
50
51
  set_template_url!(options)
51
52
  show_options(options, "cfn.update_stack")
@@ -10,7 +10,7 @@ module Lono
10
10
  long_desc Help.text(:generate)
11
11
  option :resource, default: "PretendResource", desc: "Set the @resource instance variable availalbe in the configset"
12
12
  def generate(configset)
13
- Generator.new(options.merge(configset: configset)).run
13
+ Generator.new(options.merge(configset: configset, cli: true)).run
14
14
  end
15
15
  end
16
16
  end
@@ -56,7 +56,7 @@ class Lono::Configset
56
56
 
57
57
  data = {
58
58
  registry: Lono::Jade::Registry.new(["#{logical_id}OriginalConfigset"], resource: logical_id),
59
- metdata_configset: attributes["Metadata"]
59
+ metdata_configset: {"Metadata" => attributes["Metadata"]} # # wrap metadata to create right structure
60
60
  }
61
61
  configsets << data
62
62
  end
@@ -72,40 +72,67 @@ class Lono::Configset
72
72
  end
73
73
 
74
74
  metadata_map = {}
75
+ configsets_map = {}
75
76
 
76
77
  @sets.each_with_index do |array, i|
77
78
  padded_i = "%03d" % i
78
79
  registry, metadata = array
79
- name, resource = registry.name, registry.resource
80
80
 
81
- metadata_map[resource] ||= {"AWS::CloudFormation::Init" => {"configSets" => {}}}
82
- configSets = metadata_map[resource]["AWS::CloudFormation::Init"]["configSets"]
81
+ # metadata example (full structure):
82
+ #
83
+ # {"Metadata"=>
84
+ # {"AWS::CloudFormation::Init"=>
85
+ # {"configSets"=>{"default"=>["aaa1", "aaa2"]},
86
+ # "aaa1"=>{"commands"=>{"test"=>{"command"=>"echo from-aaa1 > test1.txt"}}},
87
+ # "aaa2"=>
88
+ # {"commands"=>{"test"=>{"command"=>"echo from-aaa2 > test1.txt"}}}}}}
83
89
 
84
- configSets["default"] ||= []
85
- configSets["default"] << {"ConfigSet" => name}
90
+ name, resource = registry.name, registry.resource
91
+ configsets = configsets_map[resource] ||= {}
86
92
 
87
93
  validate_structure!(name, metadata)
88
- init = metadata["AWS::CloudFormation::Init"]
94
+
95
+ new_metadata = metadata["Metadata"].dup
96
+ init = new_metadata["AWS::CloudFormation::Init"] # important: adjust data by reference
89
97
 
90
98
  if init.key?("configSets")
99
+ validate_simple!(registry, new_metadata["AWS::CloudFormation::Init"]["configSets"]) # validate original configset for only simple elements
100
+
101
+ # 1. expand each config as its own config, flattening to top-level
91
102
  cs = init.delete("configSets") # Only support configSets with simple Array of Strings
92
- validate_simple!(registry, cs)
93
- configSets[name] = cs["default"].map {|c| "#{padded_i}_#{c}" }
103
+ new_config_set = {}
104
+ new_config_set[name] = cs["default"].map {|c| "#{padded_i}_#{c}" }
94
105
  init.transform_keys! { |c| "#{padded_i}_#{c}" }
106
+
107
+ # Rebuild default configSet, append the new complex ConfigSet structure with each iteration
108
+ configsets["default"] ||= []
109
+ configsets["default"] << {"ConfigSet" => name}
110
+ configsets.merge!(new_config_set) # add each config from #1 to the top-level
111
+
112
+ init["configSets"] = configsets # replace new configset
95
113
  else # simple config
114
+ init["configSets"] = configsets # adjust data by reference
115
+ configsets["default"] ||= []
116
+ configsets["default"] << {"ConfigSet" => name}
117
+
118
+ # build new config
96
119
  config_key = "#{padded_i}_single_generated"
97
- configSets[name] = [config_key]
98
- init = {config_key => init["config"]}
120
+ configsets[name] = [config_key]
121
+ new_config = {config_key => init["config"]}
122
+ # replace old config with new one
123
+ init.delete("config") # delete original simple config
124
+ init.merge!(new_config)
99
125
  end
100
126
 
101
- metadata_map[resource]["AWS::CloudFormation::Init"].merge!(init)
127
+ metadata_map[resource] ||= {"Metadata" => {}}
128
+ metadata_map[resource]["Metadata"].deep_merge!(new_metadata)
102
129
  @map[resource] = metadata_map[resource]
103
130
  end
104
131
  @map
105
132
  end
106
133
 
107
134
  def validate_structure!(name, metadata)
108
- return if metadata.is_a?(Hash) && metadata.key?("AWS::CloudFormation::Init")
135
+ return if metadata.is_a?(Hash) && metadata.dig("Metadata", "AWS::CloudFormation::Init")
109
136
 
110
137
  puts "ERROR: The #{name} configset does not appear to have a AWS::CloudFormation::Init key".color(:red)
111
138
  puts "Please double check the #{name} configset.yml structure"