lono 7.3.2 → 7.4.0

data/lib/lono/extensions/loader.rb

@@ -10,8 +10,9 @@ class Lono::Extensions
 
     def load_extension_helpers(registry)
       root = find_extensions_root_path(registry)
+      extension_file = "#{root}/lib/#{registry.name}"
+      require extension_file
       helpers_path = "#{root}/lib/#{registry.name}/helpers"
-
       load_files(helpers_path)
     end
 

data/lib/lono/file_uploader.rb

@@ -1,127 +1,21 @@
 module Lono
   class FileUploader < AbstractBase
-    include Lono::Template::AwsService
     extend Memoist
 
     def initialize(options={})
       super
       @checksums = {}
-      @prefix = "#{folder_key}/#{Lono.env}/#{blueprint}/files" # s3://s3-bucket/folder/development/files
+      @prefix = "#{Lono.env}/#{blueprint}/files" # s3://s3-bucket/folder/development/files
     end
 
     def upload_all
-      # pattern = "#{Lono.blueprint_root}/app/files/**/*"
-      # size = Dir.glob(pattern).size
-      # puts "size #{size}"
-      # return unless Dir.glob(pattern).size > 0
-      # puts "Uploading app/files2..."
+      puts "Uploading app/files.."
       load_checksums!
 
       Dir.glob(pattern).each do |path|
         next if ::File.directory?(path)
-        s3_upload(path)
+        Lono::S3::Uploader.new(path).upload
       end
     end
-
-    def load_checksums!
-      resp = s3.list_objects(bucket: s3_bucket, prefix: @prefix)
-      resp.contents.each do |object|
-        # key does not include the bucket name
-        #    full path = s3://my-bucket/s3_folder/files/production/my-template.yml
-        #    key = s3_folder/files/production/my-template.yml
-        # etag is the checksum as long as the file is not a multi-part file upload
-        # it has extra double quotes wrapped around it.
-        #    etag = "\"9cb437490cee2cc96101baf326e5ca81\""
-        @checksums[object.key] = strip_surrounding_quotes(object.etag)
-      end
-      @checksums
-    end
-
-    # used for file_s3_key helper
-    def md5(path)
-      Digest::MD5.file(path).to_s[0..7]
-    end
-    memoize :md5
-
-    def md5_key(path)
-      pretty_path = path.sub(/^\.\//, '')
-      key = "#{@prefix}/#{pretty_path.sub(%r{app/files/},'')}"
-      # add the short md5sum to the file
-      key = key.sub(/\.(\w+)$/,'') # strip extension
-      ext = $1
-      md5 = md5(path)
-      "#{key}-#{md5}.#{ext}"
-    end
-
-    # Inputs:
-    #
-    #   path: can be full path or relative path
-    #
-    def s3_upload(path)
-      path = path.gsub("#{Lono.root}/",'') # remove Lono.root
-      pretty_path = path.sub(/^\.\//, '')
-      key = md5_key(path)
-      s3_full_path = "s3://#{s3_bucket}/#{key}"
-
-      local_checksum = Digest::MD5.hexdigest(IO.read(path))
-      remote_checksum = remote_checksum(key)
-      if local_checksum == remote_checksum
-        puts("Not modified: #{pretty_path} to #{s3_full_path}".color(:yellow)) unless @options[:noop]
-        return # do not upload unless the checksum has changed
-      else
-        # Example output:
-        # Uploaded: app/files/docker.yml to s3://boltops-dev/s3_folder/templates/development/docker.yml
-        # Uploaded: app/files/ecs/private.yml to s3://boltops-dev/s3_folder/templates/development/ecs/private.yml
-        message = "Uploading: #{pretty_path} to #{s3_full_path}".color(:green)
-        message = "NOOP: #{message}" if @options[:noop]
-        puts message
-      end
-
-      resp = s3.put_object(
-        body: IO.read(path),
-        bucket: s3_bucket,
-        key: key,
-        storage_class: "REDUCED_REDUNDANCY"
-      ) unless @options[:noop]
-    end
-
-    # key example: cloudformation/production/files/lifecycle-0719ab81.zip
-    # s3 path: s3://boltops-dev/cloudformation/production/files/lifecycle-0719ab81.zip
-    # s3_folder: s3://boltops-dev/cloudformation
-    def remote_checksum(key)
-      @checksums[key]
-    end
-
-    def s3_bucket
-      s3_folder.sub('s3://','').split('/').first
-    end
-
-    def s3_folder
-      setting = Lono::Setting.new
-      setting.s3_folder
-    end
-
-    # The folder_key is the s3_folder setting with the s3 bucket.
-    #
-    # Example:
-    #    s3_bucket('s3://mybucket/files/storage/path')
-    #    => files/storage/path
-    def folder_key
-      return nil if @options[:noop] # to get spec passing
-      return nil unless s3_folder
-      s3_folder.sub('s3://','').split('/')[1..-1].join('/')
-    end
-
-    def strip_surrounding_quotes(string)
-      string.sub(/^"/,'').sub(/"$/,'')
-    end
-
-    def s3_resource
-      @s3_resource ||= Aws::S3::Resource.new
-    end
-
-    def s3
-      @s3 ||= Aws::S3::Client.new
-    end
   end
 end

data/lib/lono/finder/base.rb

@@ -79,7 +79,7 @@ module Lono::Finder
 
       components = find_all.sort_by { |jadespec| jadespec.name }
       components.each do |jadespec|
-        pretty_path = jadespec.root.sub("#{Lono.root}/", "")
+        pretty_path = jadespec.root.sub("#{Lono.root}/", "").sub(ENV['HOME'], "~")
         unless options[:filter_materialized] && jadespec.source_type == "materialized"
           table.rows << [jadespec.name, pretty_path, jadespec.source_type]
         end

data/lib/lono/finder/blueprint/configset.rb

@@ -1,15 +1,15 @@
 class Lono::Finder::Blueprint
   class Configset < Lono::Finder::Configset
     def initialize(options={})
-      super
+      super(**options) # **options to remove: warning: Using the last argument as keyword parameters is deprecated; maybe ** should be added to the call
       @blueprint_root = options[:blueprint_root] || Lono.blueprint_root
     end
 
     def local
-      blueprint + vendor + gems
+      project + vendor + blueprint_app + gems
     end
 
-    def blueprint
+    def blueprint_app
       roots = path_roots("#{@blueprint_root}/app/#{type.pluralize}")
       components(roots, "blueprint")
     end

data/lib/lono/generate.rb

@@ -23,6 +23,7 @@ module Lono
     def post_generate
       return if @options[:noop]
       return if @options[:source]
+
       build_files # builds app/files to output/BLUEPRINT/files
       post_process_template
       upload_files
@@ -51,10 +52,18 @@ module Lono
 
     def build_files
       Lono::AppFile::Build.new(@options).run
+      Lono::Configset::S3File::Build.new(@options).run # copies files to the output folder
     end
 
     def generate_templates
-      Lono::Template::Generator.new(@options).run
+      Lono::Template::Generator.new(@options).run # write templates to disk
+      inject_configsets
+    end
+
+    def inject_configsets
+      # The inject_configsets reads it back from disk. Leaving as-is instead of reading all in memory in case there's a reason.
+      Lono::Configset::Preparer.new(@options).run # register and materialize gems
+      Lono::Template::ConfigsetInjector.new(@options).run
     end
 
     def post_process_template
@@ -71,6 +80,7 @@ module Lono
 
     def upload_files
       Lono::AppFile::Upload.new(@options).upload
+      Lono::Configset::S3File::Upload.new(@options).upload
     end
 
     def check_for_errors

data/lib/lono/jade/registry.rb

@@ -40,11 +40,20 @@ class Lono::Jade
     class << self
       def register_configset(args, options)
         registry = new(args, options)
-        jade = Lono::Jade.new(registry.name, 'configset', registry)
+        jade_type = determine_jade_type(caller)
+        jade = Lono::Jade.new(registry.name, jade_type, registry)
         self.tracked_configsets << jade
         registry
       end
 
+      def determine_jade_type(caller)
+        if caller.detect { |l| l =~ %r{config/configsets.rb} }
+          'blueprint/configset'
+        else
+          'configset'
+        end
+      end
+
       def register_extension(args, options)
         registry = new(args, options)
         self.tracked_extensions << registry

data/lib/lono/jadespec.rb

@@ -4,14 +4,16 @@ module Lono
   class Jadespec
     extend Memoist
 
-    delegate :name, to: :gemspec
-
     attr_accessor :from
     attr_reader :root, :source_type
     def initialize(root, source_type)
       @root, @source_type = root, source_type
     end
 
+    def name
+      exist? ? gemspec.name : "not gemspec file found for @root: #{@root}"
+    end
+
     def exist?
       !!gemspec_file
     end

data/lib/lono/registration/temp.rb

@@ -29,8 +29,10 @@ class Lono::Registration
       end
 
       puts <<~EOL
-        Lono is not registered. To remove this prompt, please set up your registration
-        info in .lono/registration.yml. Registration is free. You can register at:
+        Lono is not registered. This prompt appears every 24 hours when lono is not registered.
+        To remove this prompt, please set up your registration info in .lono/registration.yml.
+
+        Registration is free. You can register at:
 
             https://register.lono.cloud
 

data/lib/lono/s3/uploader.rb

@@ -0,0 +1,94 @@
+class Lono::S3
+  class Uploader
+    include Lono::AwsServices
+    extend Memoist
+
+    attr_reader :path
+    def initialize(path, options={})
+      @path, @options = path, options
+      @checksums = {}
+    end
+
+    # Inputs:
+    #
+    #   path: can be full path or relative path
+    #
+    def upload
+      return if @options[:noop] || ENV['LONO_TEST'] == '1'
+
+      path = @path.gsub("#{Lono.root}/",'') # remove Lono.root
+      key = "#{Lono.env}/#{path}"
+
+      pretty_path = path.sub(/^\.\//, '')
+      s3_full_path = "s3://#{s3_bucket}/#{key}"
+
+      local_checksum = Digest::MD5.hexdigest(IO.read(path))
+      remote_checksum = remote_checksum(key)
+      if local_checksum == remote_checksum
+        puts("Not modified: #{pretty_path} to #{s3_full_path}".color(:yellow)) unless @options[:noop]
+        return # do not upload unless the checksum has changed
+      else
+        # Example output:
+        # Uploaded: app/files/docker.yml to s3://boltops-dev/s3_folder/templates/development/docker.yml
+        # Uploaded: app/files/ecs/private.yml to s3://boltops-dev/s3_folder/templates/development/ecs/private.yml
+        message = "Uploading: #{pretty_path} to #{s3_full_path}".color(:green)
+        message = "NOOP: #{message}" if @options[:noop]
+        puts message
+      end
+
+      s3.put_object(
+        body: IO.read(path),
+        bucket: s3_bucket,
+        key: key,
+      ) unless @options[:noop]
+    end
+
+    # https://s3.amazonaws.com/mybucket/development/output/blueprint/templates/blueprint.yml
+    def https_url
+      key = "#{Lono.env}/#{@path}"
+      "https://s3.amazonaws.com/#{s3_bucket}/#{key}"
+    end
+
+    def presigned_url
+      key = "#{Lono.env}/#{@path}"
+      s3_presigner.presigned_url(:get_object, bucket: s3_bucket, key: key)
+    end
+
+    # used for file_s3_key helper
+    def md5(path)
+      Digest::MD5.file(path).to_s[0..7]
+    end
+    memoize :md5
+
+    def load_checksums!(prefix)
+      resp = s3.list_objects(bucket: s3_bucket, prefix: prefix)
+      resp.contents.each do |object|
+        # key does not include the bucket name
+        #    full path = s3://my-bucket/s3_folder/files/production/my-template.yml
+        #    key = s3_folder/files/production/my-template.yml
+        # etag is the checksum as long as the file is not a multi-part file upload
+        # it has extra double quotes wrapped around it.
+        #    etag = "\"9cb437490cee2cc96101baf326e5ca81\""
+        @checksums[object.key] = strip_surrounding_quotes(object.etag)
+      end
+      @checksums
+    end
+    memoize :load_checksums!
+
+    # key example: cloudformation/production/files/lifecycle-0719ab81.zip
+    # s3 path: s3://boltops-dev/cloudformation/production/files/lifecycle-0719ab81.zip
+    # s3_folder: s3://boltops-dev/cloudformation
+    def remote_checksum(key)
+      load_checksums!(key)
+      @checksums[key]
+    end
+
+    def strip_surrounding_quotes(string)
+      string.sub(/^"/,'').sub(/"$/,'')
+    end
+
+    def s3_bucket
+      Lono::S3::Bucket.name
+    end
+  end
+end

data/lib/lono/seed/base.rb

@@ -57,7 +57,8 @@ class Lono::Seed
       @output_template.parameter_groups.each do |label, parameters|
         lines << "# Parameter Group: #{label}"
         parameters.each do |name|
-          lines << parameter_line(name)
+          parameter_line = parameter_line(name)
+          lines << parameter_line unless lines.include?(parameter_line)
           shown << name
         end
         lines << ""

data/lib/lono/template/configset_injector.rb

@@ -20,10 +20,12 @@ class Lono::Template
           next
         end
 
-        metdata = resource["Metadata"] ||= {}
-        metdata["AWS::CloudFormation::Init"] ||= {}
-        # The metadata_configset has been combined with the original AWS::CloudFormation::Init if it exists
-        metdata["AWS::CloudFormation::Init"] = metadata_configset["AWS::CloudFormation::Init"]
+        resource["Metadata"] ||= metadata_configset["Metadata"]
+
+        # metdata = resource["Metadata"] ||= {}
+        # metdata["AWS::CloudFormation::Init"] ||= {}
+        # # The metadata_configset has been combined with the original AWS::CloudFormation::Init if it exists
+        # metdata["AWS::CloudFormation::Init"] = metadata_configset["AWS::CloudFormation::Init"]
       end
 
       @cfn

data/lib/lono/template/generator.rb

@@ -8,10 +8,7 @@ class Lono::Template
       #   Dsl.new(options).run
       generator_class = "Lono::Template::Strategy::#{lono_strategy.camelize}"
       generator_class = Object.const_get(generator_class)
-      generator_class.new(@options).run
-      # The generator strategy class writes template to disk. The inject_configsets reads it back from disk.
-      # Leaving as-is instead of reading all in memory in case there's a reason.
-      inject_configsets
+      generator_class.new(@options).run # The generator strategy classes write templates to disk
     end
 
     def lono_strategy
@@ -22,10 +19,5 @@ class Lono::Template
         jadespec.lono_strategy
       end
     end
-
-    def inject_configsets
-      Lono::Configset::Preparer.new(@options).run # register and materialize gems
-      ConfigsetInjector.new(@options).run
-    end
   end
 end

data/lib/lono/template/helper.rb

@@ -8,10 +8,9 @@ require "aws-sdk-core"
 #   Lono::Template::Context.new(blueprint, @options)
 module Lono::Template::Helper
   def template_s3_path(template_name)
-    # high jacking Upload for useful s3_https_url method
+    # hi-jacking Uploader for https_url
     template_path = "output/#{@blueprint}/templates/#{template_name}.yml"
-    upload = Lono::Template::Upload.new(@options)
-    upload.s3_https_url(template_path)
+    Lono::S3::Uploader.new(template_path).s3_https_url
   end
 
   def template_params(param_name)

data/lib/lono/template/post_processor.rb

@@ -13,17 +13,25 @@ class Lono::Template
 
     def replacements
       map = {}
+
       registry_items.each do |item|
         if item.type == "lambda_layer"
           placeholder = "file://app/files/lambda_layer/#{item.name}"
         elsif item.directory? || item.file?
           placeholder = "file://app/files/file/#{item.name}"
         else
-          puts "WARN: PostProcessor replacements Cannot find file: #{item.path}"
+          puts "WARN: PostProcessor replacements Cannot find file: #{item.output_path}"
           next
         end
         map[placeholder] = item.s3_path
       end
+
+      Lono::Configset::S3File::Registry.items.each do |item|
+        placeholder = "file://configset/#{item.configset}/#{item.name}"
+        # map[placeholder] = "https://s3.amazonaws.com/#{Lono::S3::Bucket.name}/#{item.s3_path}"
+        map[placeholder] = item.replacement_value
+      end
+
       map
     end