lono 7.3.2 → 7.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +4 -0
  3. data/lib/lono.rb +1 -0
  4. data/lib/lono/app_file/build.rb +22 -34
  5. data/lib/lono/app_file/build/lambda_layer/ruby_packager.rb +2 -33
  6. data/lib/lono/app_file/registry/item.rb +18 -45
  7. data/lib/lono/app_file/upload.rb +1 -25
  8. data/lib/lono/cfn/base.rb +8 -2
  9. data/lib/lono/cfn/create.rb +3 -2
  10. data/lib/lono/cfn/opts.rb +8 -0
  11. data/lib/lono/cfn/preview/changeset.rb +1 -0
  12. data/lib/lono/cfn/update.rb +1 -0
  13. data/lib/lono/configset.rb +1 -1
  14. data/lib/lono/configset/combiner.rb +40 -13
  15. data/lib/lono/configset/generator.rb +6 -1
  16. data/lib/lono/configset/list.rb +1 -1
  17. data/lib/lono/configset/register/dsl.rb +3 -3
  18. data/lib/lono/configset/resolver.rb +9 -5
  19. data/lib/lono/configset/s3_file/build.rb +33 -0
  20. data/lib/lono/configset/s3_file/item.rb +30 -0
  21. data/lib/lono/configset/s3_file/registry.rb +12 -0
  22. data/lib/lono/configset/s3_file/upload.rb +12 -0
  23. data/lib/lono/configset/strategy/base.rb +21 -1
  24. data/lib/lono/configset/strategy/dsl.rb +9 -6
  25. data/lib/lono/configset/strategy/erb.rb +6 -1
  26. data/lib/lono/configset/strategy/helpers/dsl.rb +8 -0
  27. data/lib/lono/configset/strategy/helpers/dsl/auth.rb +37 -0
  28. data/lib/lono/configset/strategy/{dsl/helpers → helpers/dsl}/core.rb +6 -1
  29. data/lib/lono/configset/strategy/helpers/dsl/package.rb +40 -0
  30. data/lib/lono/configset/strategy/helpers/dsl/syntax.rb +74 -0
  31. data/lib/lono/configset/strategy/{erb/helpers.rb → helpers/erb.rb} +2 -2
  32. data/lib/lono/extension/list.rb +2 -1
  33. data/lib/lono/extensions/loader.rb +2 -1
  34. data/lib/lono/file_uploader.rb +3 -109
  35. data/lib/lono/finder/base.rb +1 -1
  36. data/lib/lono/finder/blueprint/configset.rb +3 -3
  37. data/lib/lono/generate.rb +11 -1
  38. data/lib/lono/jade/registry.rb +10 -1
  39. data/lib/lono/jadespec.rb +4 -2
  40. data/lib/lono/registration/temp.rb +4 -2
  41. data/lib/lono/s3/uploader.rb +94 -0
  42. data/lib/lono/seed/base.rb +2 -1
  43. data/lib/lono/template/configset_injector.rb +6 -4
  44. data/lib/lono/template/generator.rb +1 -9
  45. data/lib/lono/template/helper.rb +2 -3
  46. data/lib/lono/template/post_processor.rb +9 -1
  47. data/lib/lono/template/strategy/dsl/builder/helpers.rb +1 -0
  48. data/lib/lono/template/strategy/dsl/builder/helpers/ec2_helper.rb +27 -0
  49. data/lib/lono/template/strategy/dsl/builder/helpers/s3_helper.rb +1 -0
  50. data/lib/lono/template/upload.rb +4 -98
  51. data/lib/lono/utils/item/file_methods.rb +29 -0
  52. data/lib/lono/utils/item/zip.rb +42 -0
  53. data/lib/lono/utils/rsync.rb +36 -0
  54. data/lib/lono/version.rb +1 -1
  55. data/lib/templates/blueprint/.gitignore +4 -7
  56. data/lib/templates/configset/%configset_name%.gemspec.tt +1 -1
  57. data/lib/templates/configset/.gitignore +2 -4
  58. data/lib/templates/extension/%extension_name%.gemspec.tt +2 -1
  59. data/lib/templates/extension/.gitignore +11 -17
  60. data/lib/templates/extension/lib/%extension_name%.rb.tt +3 -0
  61. data/lib/templates/extension/lib/%extension_name%/autoloader.rb.tt +23 -0
  62. data/lono.gemspec +1 -0
  63. metadata +32 -6
  64. data/lib/lono/configset/strategy/dsl/helpers.rb +0 -5
  65. data/lib/lono/configset/strategy/dsl/syntax.rb +0 -32
@@ -10,8 +10,9 @@ class Lono::Extensions
10
10
 
11
11
  def load_extension_helpers(registry)
12
12
  root = find_extensions_root_path(registry)
13
+ extension_file = "#{root}/lib/#{registry.name}"
14
+ require extension_file
13
15
  helpers_path = "#{root}/lib/#{registry.name}/helpers"
14
-
15
16
  load_files(helpers_path)
16
17
  end
17
18
 
@@ -1,127 +1,21 @@
1
1
  module Lono
2
2
  class FileUploader < AbstractBase
3
- include Lono::Template::AwsService
4
3
  extend Memoist
5
4
 
6
5
  def initialize(options={})
7
6
  super
8
7
  @checksums = {}
9
- @prefix = "#{folder_key}/#{Lono.env}/#{blueprint}/files" # s3://s3-bucket/folder/development/files
8
+ @prefix = "#{Lono.env}/#{blueprint}/files" # s3://s3-bucket/folder/development/files
10
9
  end
11
10
 
12
11
  def upload_all
13
- # pattern = "#{Lono.blueprint_root}/app/files/**/*"
14
- # size = Dir.glob(pattern).size
15
- # puts "size #{size}"
16
- # return unless Dir.glob(pattern).size > 0
17
- # puts "Uploading app/files2..."
12
+ puts "Uploading app/files.."
18
13
  load_checksums!
19
14
 
20
15
  Dir.glob(pattern).each do |path|
21
16
  next if ::File.directory?(path)
22
- s3_upload(path)
17
+ Lono::S3::Uploader.new(path).upload
23
18
  end
24
19
  end
25
-
26
- def load_checksums!
27
- resp = s3.list_objects(bucket: s3_bucket, prefix: @prefix)
28
- resp.contents.each do |object|
29
- # key does not include the bucket name
30
- # full path = s3://my-bucket/s3_folder/files/production/my-template.yml
31
- # key = s3_folder/files/production/my-template.yml
32
- # etag is the checksum as long as the file is not a multi-part file upload
33
- # it has extra double quotes wrapped around it.
34
- # etag = "\"9cb437490cee2cc96101baf326e5ca81\""
35
- @checksums[object.key] = strip_surrounding_quotes(object.etag)
36
- end
37
- @checksums
38
- end
39
-
40
- # used for file_s3_key helper
41
- def md5(path)
42
- Digest::MD5.file(path).to_s[0..7]
43
- end
44
- memoize :md5
45
-
46
- def md5_key(path)
47
- pretty_path = path.sub(/^\.\//, '')
48
- key = "#{@prefix}/#{pretty_path.sub(%r{app/files/},'')}"
49
- # add the short md5sum to the file
50
- key = key.sub(/\.(\w+)$/,'') # strip extension
51
- ext = $1
52
- md5 = md5(path)
53
- "#{key}-#{md5}.#{ext}"
54
- end
55
-
56
- # Inputs:
57
- #
58
- # path: can be full path or relative path
59
- #
60
- def s3_upload(path)
61
- path = path.gsub("#{Lono.root}/",'') # remove Lono.root
62
- pretty_path = path.sub(/^\.\//, '')
63
- key = md5_key(path)
64
- s3_full_path = "s3://#{s3_bucket}/#{key}"
65
-
66
- local_checksum = Digest::MD5.hexdigest(IO.read(path))
67
- remote_checksum = remote_checksum(key)
68
- if local_checksum == remote_checksum
69
- puts("Not modified: #{pretty_path} to #{s3_full_path}".color(:yellow)) unless @options[:noop]
70
- return # do not upload unless the checksum has changed
71
- else
72
- # Example output:
73
- # Uploaded: app/files/docker.yml to s3://boltops-dev/s3_folder/templates/development/docker.yml
74
- # Uploaded: app/files/ecs/private.yml to s3://boltops-dev/s3_folder/templates/development/ecs/private.yml
75
- message = "Uploading: #{pretty_path} to #{s3_full_path}".color(:green)
76
- message = "NOOP: #{message}" if @options[:noop]
77
- puts message
78
- end
79
-
80
- resp = s3.put_object(
81
- body: IO.read(path),
82
- bucket: s3_bucket,
83
- key: key,
84
- storage_class: "REDUCED_REDUNDANCY"
85
- ) unless @options[:noop]
86
- end
87
-
88
- # key example: cloudformation/production/files/lifecycle-0719ab81.zip
89
- # s3 path: s3://boltops-dev/cloudformation/production/files/lifecycle-0719ab81.zip
90
- # s3_folder: s3://boltops-dev/cloudformation
91
- def remote_checksum(key)
92
- @checksums[key]
93
- end
94
-
95
- def s3_bucket
96
- s3_folder.sub('s3://','').split('/').first
97
- end
98
-
99
- def s3_folder
100
- setting = Lono::Setting.new
101
- setting.s3_folder
102
- end
103
-
104
- # The folder_key is the s3_folder setting with the s3 bucket.
105
- #
106
- # Example:
107
- # s3_bucket('s3://mybucket/files/storage/path')
108
- # => files/storage/path
109
- def folder_key
110
- return nil if @options[:noop] # to get spec passing
111
- return nil unless s3_folder
112
- s3_folder.sub('s3://','').split('/')[1..-1].join('/')
113
- end
114
-
115
- def strip_surrounding_quotes(string)
116
- string.sub(/^"/,'').sub(/"$/,'')
117
- end
118
-
119
- def s3_resource
120
- @s3_resource ||= Aws::S3::Resource.new
121
- end
122
-
123
- def s3
124
- @s3 ||= Aws::S3::Client.new
125
- end
126
20
  end
127
21
  end
@@ -79,7 +79,7 @@ module Lono::Finder
79
79
 
80
80
  components = find_all.sort_by { |jadespec| jadespec.name }
81
81
  components.each do |jadespec|
82
- pretty_path = jadespec.root.sub("#{Lono.root}/", "")
82
+ pretty_path = jadespec.root.sub("#{Lono.root}/", "").sub(ENV['HOME'], "~")
83
83
  unless options[:filter_materialized] && jadespec.source_type == "materialized"
84
84
  table.rows << [jadespec.name, pretty_path, jadespec.source_type]
85
85
  end
@@ -1,15 +1,15 @@
1
1
  class Lono::Finder::Blueprint
2
2
  class Configset < Lono::Finder::Configset
3
3
  def initialize(options={})
4
- super
4
+ super(**options) # **options to remove: warning: Using the last argument as keyword parameters is deprecated; maybe ** should be added to the call
5
5
  @blueprint_root = options[:blueprint_root] || Lono.blueprint_root
6
6
  end
7
7
 
8
8
  def local
9
- blueprint + vendor + gems
9
+ project + vendor + blueprint_app + gems
10
10
  end
11
11
 
12
- def blueprint
12
+ def blueprint_app
13
13
  roots = path_roots("#{@blueprint_root}/app/#{type.pluralize}")
14
14
  components(roots, "blueprint")
15
15
  end
@@ -23,6 +23,7 @@ module Lono
23
23
  def post_generate
24
24
  return if @options[:noop]
25
25
  return if @options[:source]
26
+
26
27
  build_files # builds app/files to output/BLUEPRINT/files
27
28
  post_process_template
28
29
  upload_files
@@ -51,10 +52,18 @@ module Lono
51
52
 
52
53
  def build_files
53
54
  Lono::AppFile::Build.new(@options).run
55
+ Lono::Configset::S3File::Build.new(@options).run # copies files to the output folder
54
56
  end
55
57
 
56
58
  def generate_templates
57
- Lono::Template::Generator.new(@options).run
59
+ Lono::Template::Generator.new(@options).run # write templates to disk
60
+ inject_configsets
61
+ end
62
+
63
+ def inject_configsets
64
+ # The inject_configsets reads it back from disk. Leaving as-is instead of reading all in memory in case there's a reason.
65
+ Lono::Configset::Preparer.new(@options).run # register and materialize gems
66
+ Lono::Template::ConfigsetInjector.new(@options).run
58
67
  end
59
68
 
60
69
  def post_process_template
@@ -71,6 +80,7 @@ module Lono
71
80
 
72
81
  def upload_files
73
82
  Lono::AppFile::Upload.new(@options).upload
83
+ Lono::Configset::S3File::Upload.new(@options).upload
74
84
  end
75
85
 
76
86
  def check_for_errors
@@ -40,11 +40,20 @@ class Lono::Jade
40
40
  class << self
41
41
  def register_configset(args, options)
42
42
  registry = new(args, options)
43
- jade = Lono::Jade.new(registry.name, 'configset', registry)
43
+ jade_type = determine_jade_type(caller)
44
+ jade = Lono::Jade.new(registry.name, jade_type, registry)
44
45
  self.tracked_configsets << jade
45
46
  registry
46
47
  end
47
48
 
49
+ def determine_jade_type(caller)
50
+ if caller.detect { |l| l =~ %r{config/configsets.rb} }
51
+ 'blueprint/configset'
52
+ else
53
+ 'configset'
54
+ end
55
+ end
56
+
48
57
  def register_extension(args, options)
49
58
  registry = new(args, options)
50
59
  self.tracked_extensions << registry
@@ -4,14 +4,16 @@ module Lono
4
4
  class Jadespec
5
5
  extend Memoist
6
6
 
7
- delegate :name, to: :gemspec
8
-
9
7
  attr_accessor :from
10
8
  attr_reader :root, :source_type
11
9
  def initialize(root, source_type)
12
10
  @root, @source_type = root, source_type
13
11
  end
14
12
 
13
+ def name
14
+ exist? ? gemspec.name : "not gemspec file found for @root: #{@root}"
15
+ end
16
+
15
17
  def exist?
16
18
  !!gemspec_file
17
19
  end
@@ -29,8 +29,10 @@ class Lono::Registration
29
29
  end
30
30
 
31
31
  puts <<~EOL
32
- Lono is not registered. To remove this prompt, please set up your registration
33
- info in .lono/registration.yml. Registration is free. You can register at:
32
+ Lono is not registered. This prompt appears every 24 hours when lono is not registered.
33
+ To remove this prompt, please set up your registration info in .lono/registration.yml.
34
+
35
+ Registration is free. You can register at:
34
36
 
35
37
  https://register.lono.cloud
36
38
 
@@ -0,0 +1,94 @@
1
+ class Lono::S3
2
+ class Uploader
3
+ include Lono::AwsServices
4
+ extend Memoist
5
+
6
+ attr_reader :path
7
+ def initialize(path, options={})
8
+ @path, @options = path, options
9
+ @checksums = {}
10
+ end
11
+
12
+ # Inputs:
13
+ #
14
+ # path: can be full path or relative path
15
+ #
16
+ def upload
17
+ return if @options[:noop] || ENV['LONO_TEST'] == '1'
18
+
19
+ path = @path.gsub("#{Lono.root}/",'') # remove Lono.root
20
+ key = "#{Lono.env}/#{path}"
21
+
22
+ pretty_path = path.sub(/^\.\//, '')
23
+ s3_full_path = "s3://#{s3_bucket}/#{key}"
24
+
25
+ local_checksum = Digest::MD5.hexdigest(IO.read(path))
26
+ remote_checksum = remote_checksum(key)
27
+ if local_checksum == remote_checksum
28
+ puts("Not modified: #{pretty_path} to #{s3_full_path}".color(:yellow)) unless @options[:noop]
29
+ return # do not upload unless the checksum has changed
30
+ else
31
+ # Example output:
32
+ # Uploaded: app/files/docker.yml to s3://boltops-dev/s3_folder/templates/development/docker.yml
33
+ # Uploaded: app/files/ecs/private.yml to s3://boltops-dev/s3_folder/templates/development/ecs/private.yml
34
+ message = "Uploading: #{pretty_path} to #{s3_full_path}".color(:green)
35
+ message = "NOOP: #{message}" if @options[:noop]
36
+ puts message
37
+ end
38
+
39
+ s3.put_object(
40
+ body: IO.read(path),
41
+ bucket: s3_bucket,
42
+ key: key,
43
+ ) unless @options[:noop]
44
+ end
45
+
46
+ # https://s3.amazonaws.com/mybucket/development/output/blueprint/templates/blueprint.yml
47
+ def https_url
48
+ key = "#{Lono.env}/#{@path}"
49
+ "https://s3.amazonaws.com/#{s3_bucket}/#{key}"
50
+ end
51
+
52
+ def presigned_url
53
+ key = "#{Lono.env}/#{@path}"
54
+ s3_presigner.presigned_url(:get_object, bucket: s3_bucket, key: key)
55
+ end
56
+
57
+ # used for file_s3_key helper
58
+ def md5(path)
59
+ Digest::MD5.file(path).to_s[0..7]
60
+ end
61
+ memoize :md5
62
+
63
+ def load_checksums!(prefix)
64
+ resp = s3.list_objects(bucket: s3_bucket, prefix: prefix)
65
+ resp.contents.each do |object|
66
+ # key does not include the bucket name
67
+ # full path = s3://my-bucket/s3_folder/files/production/my-template.yml
68
+ # key = s3_folder/files/production/my-template.yml
69
+ # etag is the checksum as long as the file is not a multi-part file upload
70
+ # it has extra double quotes wrapped around it.
71
+ # etag = "\"9cb437490cee2cc96101baf326e5ca81\""
72
+ @checksums[object.key] = strip_surrounding_quotes(object.etag)
73
+ end
74
+ @checksums
75
+ end
76
+ memoize :load_checksums!
77
+
78
+ # key example: cloudformation/production/files/lifecycle-0719ab81.zip
79
+ # s3 path: s3://boltops-dev/cloudformation/production/files/lifecycle-0719ab81.zip
80
+ # s3_folder: s3://boltops-dev/cloudformation
81
+ def remote_checksum(key)
82
+ load_checksums!(key)
83
+ @checksums[key]
84
+ end
85
+
86
+ def strip_surrounding_quotes(string)
87
+ string.sub(/^"/,'').sub(/"$/,'')
88
+ end
89
+
90
+ def s3_bucket
91
+ Lono::S3::Bucket.name
92
+ end
93
+ end
94
+ end
@@ -57,7 +57,8 @@ class Lono::Seed
57
57
  @output_template.parameter_groups.each do |label, parameters|
58
58
  lines << "# Parameter Group: #{label}"
59
59
  parameters.each do |name|
60
- lines << parameter_line(name)
60
+ parameter_line = parameter_line(name)
61
+ lines << parameter_line unless lines.include?(parameter_line)
61
62
  shown << name
62
63
  end
63
64
  lines << ""
@@ -20,10 +20,12 @@ class Lono::Template
20
20
  next
21
21
  end
22
22
 
23
- metdata = resource["Metadata"] ||= {}
24
- metdata["AWS::CloudFormation::Init"] ||= {}
25
- # The metadata_configset has been combined with the original AWS::CloudFormation::Init if it exists
26
- metdata["AWS::CloudFormation::Init"] = metadata_configset["AWS::CloudFormation::Init"]
23
+ resource["Metadata"] ||= metadata_configset["Metadata"]
24
+
25
+ # metdata = resource["Metadata"] ||= {}
26
+ # metdata["AWS::CloudFormation::Init"] ||= {}
27
+ # # The metadata_configset has been combined with the original AWS::CloudFormation::Init if it exists
28
+ # metdata["AWS::CloudFormation::Init"] = metadata_configset["AWS::CloudFormation::Init"]
27
29
  end
28
30
 
29
31
  @cfn
@@ -8,10 +8,7 @@ class Lono::Template
8
8
  # Dsl.new(options).run
9
9
  generator_class = "Lono::Template::Strategy::#{lono_strategy.camelize}"
10
10
  generator_class = Object.const_get(generator_class)
11
- generator_class.new(@options).run
12
- # The generator strategy class writes template to disk. The inject_configsets reads it back from disk.
13
- # Leaving as-is instead of reading all in memory in case there's a reason.
14
- inject_configsets
11
+ generator_class.new(@options).run # The generator strategy classes write templates to disk
15
12
  end
16
13
 
17
14
  def lono_strategy
@@ -22,10 +19,5 @@ class Lono::Template
22
19
  jadespec.lono_strategy
23
20
  end
24
21
  end
25
-
26
- def inject_configsets
27
- Lono::Configset::Preparer.new(@options).run # register and materialize gems
28
- ConfigsetInjector.new(@options).run
29
- end
30
22
  end
31
23
  end
@@ -8,10 +8,9 @@ require "aws-sdk-core"
8
8
  # Lono::Template::Context.new(blueprint, @options)
9
9
  module Lono::Template::Helper
10
10
  def template_s3_path(template_name)
11
- # high jacking Upload for useful s3_https_url method
11
+ # hi-jacking Uploader for https_url
12
12
  template_path = "output/#{@blueprint}/templates/#{template_name}.yml"
13
- upload = Lono::Template::Upload.new(@options)
14
- upload.s3_https_url(template_path)
13
+ Lono::S3::Uploader.new(template_path).s3_https_url
15
14
  end
16
15
 
17
16
  def template_params(param_name)
@@ -13,17 +13,25 @@ class Lono::Template
13
13
 
14
14
  def replacements
15
15
  map = {}
16
+
16
17
  registry_items.each do |item|
17
18
  if item.type == "lambda_layer"
18
19
  placeholder = "file://app/files/lambda_layer/#{item.name}"
19
20
  elsif item.directory? || item.file?
20
21
  placeholder = "file://app/files/file/#{item.name}"
21
22
  else
22
- puts "WARN: PostProcessor replacements Cannot find file: #{item.path}"
23
+ puts "WARN: PostProcessor replacements Cannot find file: #{item.output_path}"
23
24
  next
24
25
  end
25
26
  map[placeholder] = item.s3_path
26
27
  end
28
+
29
+ Lono::Configset::S3File::Registry.items.each do |item|
30
+ placeholder = "file://configset/#{item.configset}/#{item.name}"
31
+ # map[placeholder] = "https://s3.amazonaws.com/#{Lono::S3::Bucket.name}/#{item.s3_path}"
32
+ map[placeholder] = item.replacement_value
33
+ end
34
+
27
35
  map
28
36
  end
29
37