lono 7.3.2 → 7.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eafe64d8f51c8ca12fdbca5fb6cba0643389a834bca76d1031622e643322d9c2
-  data.tar.gz: 746097bef6f91a86d8079294b9119dd3d6c6c79f707203fd716c6239a72f9198
+  metadata.gz: c0b9294948bbd81eea5198852bffe28c7682aaf9a73cba1273f9fa0b5f2fd7d7
+  data.tar.gz: 948e6cb7dd9ee9ecf5ab02c85f37687339598535bb34deff83dbf02ddc511cf4
 SHA512:
-  metadata.gz: e98615585a826093a393b8c082b4f879ada01c1f12df4e516c56e2aa662cd04b149bc113a1ea24213079799d5769bfbd7d5524f25d377703b0d58d598f3e9df6
-  data.tar.gz: 8c94729c489105f755648dde84cfa154143823208d4c52683b2f6d1e21014e618ae599ea559113292f15cf44c15dc09c99da659705015c210bec1a5647a2705a
+  metadata.gz: 7d14a3286bf4b0aba8e2a7c1129abc222b4379dd4c5f3214bcb906c08a0e078ad4455bdd68fcf7e1f774a382dd19e7e0704dacbaf0236517b5c1dce23927af38
+  data.tar.gz: f3a65caf45ca739360cae97be4f16e5eb45c18030eb537e1cbdccbfefa5484e52368ba656c057b75818e1e26bbdb093f173d72ffbab48d4d317f1b4e5dbfe594

data/CHANGELOG.md

@@ -3,6 +3,10 @@
 All notable changes to this project will be documented in this file.
 This project *tries* to adhere to [Semantic Versioning](http://semver.org/), even before v1.0.
 
+## [7.4.0]
+- #57 Big improvements generally and configsets
+- #58 Docs: Configsets and new DSL Helpers
+
 ## [7.3.2]
 - #56 build lambda layer fix: allow Gemfile.lock to be generated if not provided
 

data/lib/lono.rb

@@ -3,6 +3,7 @@ $stdout.sync = true unless ENV["LONO_STDOUT_SYNC"] == "0"
 require "active_support/core_ext/class"
 require 'active_support/core_ext/hash'
 require 'active_support/core_ext/string'
+require 'aws_data'
 require 'cfn_camelizer'
 require 'fileutils'
 require 'json'

data/lib/lono/app_file/build.rb

@@ -2,6 +2,8 @@ require "thor"
 
 module Lono::AppFile
   class Build < Base
+    include Lono::Utils::Item::Zip
+
     def initialize_variables
       @output_files_path = "#{Lono.config.output_path}/#{@blueprint}/files"
     end
@@ -14,11 +16,28 @@ module Lono::AppFile
 
     def build_all
       clean_output
+      validate_files!
       copy_to_output
       build_layers
       compress_output
     end
 
+    def validate_files!
+      items = Registry.items + Registry.layers
+      missing = items.select do |item|
+        !File.exist?(item.src_path)
+      end
+      missing_paths = missing.map { |item| item.src_path }.uniq
+      unless missing_paths.empty?
+        puts "ERROR: These app/files are missing were used by the s3_key method but are missing".color(:red)
+        missing_paths.each do |path|
+          puts "    #{path}"
+        end
+        puts "Please double check that they exist."
+        exit 1
+      end
+    end
+
     def build_layers
       layer_items = Registry.layers
       layer_items.each do |item|
@@ -29,12 +48,10 @@ module Lono::AppFile
     def compress_output
       Registry.items.each do |item|
         # type can be lambda_layer or file
-        if item.type == "lambda_layer" || item.directory?
-          zip_directory(item)
-        elsif item.file?
-          zip_file(item)
+        if item.type == "lambda_layer" || item.exist?
+          zip(item)
         else
-          puts "WARN: #{item.path} does not exist. Double check that the path is correct in the s3_key call.".color(:yellow)
+          puts "WARN: #{item.src_path} does not exist. Double check that the path is correct in the s3_key call.".color(:yellow)
         end
       end
     end
@@ -50,35 +67,6 @@ module Lono::AppFile
     end
     memoize :context
 
-    def zip_file(item)
-      path = item.path
-      zip_file = item.zip_file_name
-
-      puts "Zipping file and generating md5 named file from: #{path}"
-      command = "cd #{File.dirname(path)} && zip -q #{zip_file} #{File.basename(path)}" # create zipfile at same level of file
-      zip(command)
-    end
-
-    def zip_directory(item)
-      path = item.path
-      zip_file = item.zip_file_name
-
-      puts "Zipping folder and generating md5 named file from: #{path}"
-      command = "cd #{path} && zip --symlinks -rq #{zip_file} ." # create zipfile witih directory
-      zip(command)
-      FileUtils.mv("#{path}/#{zip_file}", "#{File.dirname(path)}/#{zip_file}") # move zip back to the parent directory
-    end
-
-    def zip(command)
-      # puts "=> #{command}".color(:green) # uncomment to debug
-      `#{command}`
-      unless $?.success?
-        puts "ERROR: Fail to run #{command}".color(:red)
-        puts "Maybe zip is not installed or path is incorrect?"
-        exit 1
-      end
-    end
-
     def clean_output
       FileUtils.rm_rf(@output_files_path)
     end

data/lib/lono/app_file/build/lambda_layer/ruby_packager.rb

@@ -1,8 +1,8 @@
-require 'shellwords'
-
 class Lono::AppFile::Build::LambdaLayer
   # Based on jets
   class RubyPackager
+    include Lono::Utils::Rsync
+
     def initialize(blueprint, registry_item)
       @blueprint, @registry_item = blueprint, registry_item
 
@@ -157,36 +157,5 @@ EOL
       gemfile_path = "#{@app_root}/Gemfile"
       File.exist?(gemfile_path)
     end
-
-    def sh(command)
-      puts "=> #{command}"
-      out = `#{command}`
-      puts out if ENV['LONO_DEBUG_LAMBDA_LAYER']
-      success = $?.success?
-      raise("ERROR: running command #{command}").color(:red) unless success
-      success
-    end
-
-    def rsync(src, dest)
-      # Using FileUtils.cp_r doesnt work if there are special files like socket files in the src dir.
-      # Instead of using this hack https://bugs.ruby-lang.org/issues/10104
-      # Using rsync to perform the copy.
-      src.chop! if src.ends_with?('/')
-      dest.chop! if dest.ends_with?('/')
-      check_rsync_installed!
-      # Ensures required trailing slashes
-      FileUtils.mkdir_p(File.dirname(dest))
-      sh "rsync -a --links --no-specials --no-devices #{Shellwords.escape(src)}/ #{Shellwords.escape(dest)}/"
-    end
-
-    @@rsync_installed = false
-    def check_rsync_installed!
-      return if @@rsync_installed # only check once
-      if system "type rsync > /dev/null 2>&1"
-        @@rsync_installed = true
-      else
-        raise "ERROR: Rsync is required. Rsync does not seem to be installed.".color(:red)
-      end
-    end
   end
 end

data/lib/lono/app_file/registry/item.rb

@@ -1,50 +1,23 @@
-module Lono::AppFile
-  class Registry
-    # Holds metadata about the item in the regsitry.
-    class Item
-      attr_reader :name, :options, :type
-      def initialize(name, blueprint, options={})
-        @name, @blueprint, @options = name, blueprint, options
-        @type = options[:type] || "file"
-      end
-
-      def path
-        if @type == "file"
-          "#{Lono.root}/output/#{@blueprint}/files/#{@name}"
-        else
-          "#{Lono.root}/output/#{@blueprint}/lambda_layers/#{@name}/opt"
-        end
-      end
-
-      def directory?
-        File.directory?(path)
-      end
-
-      def file?
-        File.file?(path)
-      end
-
-      def s3_path
-        file_path = zip_file_path.sub(%r{.*/output/[\w_-]+/(files|lambda_layers)/}, '') # dont use basename. there might be subfolders
-        "#{s3_prefix}/#{file_path}"
-      end
-
-      # full path
-      def zip_file_path
-        "#{File.dirname(path)}/#{zip_file_name}"
-      end
-
-      def zip_file_name
-        "#{File.basename(path)}-#{@type}-#{Lono::Md5.sum(path)}.zip"
-      end
+class Lono::AppFile::Registry
+  # Holds metadata about the item in the regsitry.
+  class Item
+    include Lono::Utils::Item::FileMethods
+
+    attr_reader :name, :options, :type
+    def initialize(name, blueprint, options={})
+      @name, @blueprint, @options = name, blueprint, options
+      @type = options[:type] || "file"
+    end
 
-    private
-      def md5_path
-        Lono::Md5.name(path)
-      end
+    def src_path
+      "#{Lono.blueprint_root}/app/files/#{@name}"
+    end
 
-      def s3_prefix
-        "#{Lono.env}/#{@blueprint}/files" # development/ecs-asg/files
+    def output_path
+      if @type == "file"
+        "#{Lono.root}/output/#{@blueprint}/files/#{@name}"
+      else
+        "#{Lono.root}/output/#{@blueprint}/lambda_layers/#{@name}/opt"
       end
     end
   end

data/lib/lono/app_file/upload.rb

@@ -1,36 +1,12 @@
 module Lono::AppFile
   class Upload < Base
-    include Lono::AwsServices
-    extend Memoist
-
     def upload
       return unless Registry.items.size > 0
       puts "Uploading app/files..."
 
       Registry.items.each do |item|
-        s3_upload(item)
+        Lono::S3::Uploader.new(item.zip_file_path).upload
       end
     end
-
-    # TODO: check md5sum and only upload if it changes
-    def s3_upload(item)
-      filepath = item.zip_file_path
-      s3_key = item.s3_path
-      s3_path = "s3://#{s3_bucket}/#{s3_key}"
-      message = "Uploading: #{filepath} to #{s3_path}".color(:green)
-      message = "NOOP: #{message}" if @options[:noop]
-      puts message
-      return if @options[:noop]
-
-      s3.put_object(
-        body: IO.read(filepath),
-        bucket: s3_bucket,
-        key: s3_key,
-      )
-    end
-
-    def s3_bucket
-      Lono::S3::Bucket.name
-    end
   end
 end

data/lib/lono/cfn/base.rb

@@ -167,6 +167,13 @@ class Lono::Cfn
       end
     end
 
+    def notification_arns
+      @setting ||= Lono::Setting.new
+      settings = @setting.data
+      arns = @options["notification_arns"] || settings["notification_arns"]
+      arns == [''] ? [] : arns # allow removing the notification_arns setting
+    end
+
     def show_options(options, meth=nil)
       options = options.clone.compact
       if options[:template_body] # continue_update_rollback
@@ -185,9 +192,8 @@ class Lono::Cfn
     #
     # Reference: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/cloudformation-limits.html
     def set_template_url!(options)
-      upload = Lono::Template::Upload.new(@options)
       url_path = template_path.sub("#{Lono.root}/",'')
-      url = upload.s3_presigned_url(url_path)
+      url = Lono::S3::Uploader.new(url_path).presigned_url
       url.gsub!(/\.yml.*/, ".yml") # Interesting dont need presign query string. For stack sets it actually breaks it. So removing.
       options[:template_url] = url
       options

data/lib/lono/cfn/create.rb

@@ -24,11 +24,12 @@ class Lono::Cfn
       end
 
       options = {
-        stack_name: @stack,
-        parameters: parameters,
         capabilities: capabilities, # ["CAPABILITY_IAM", "CAPABILITY_NAMED_IAM"]
         disable_rollback: !@options[:rollback],
+        parameters: parameters,
+        stack_name: @stack,
       }
+      options[:notification_arns] = notification_arns if notification_arns
       options[:tags] = tags unless tags.empty?
       set_template_url!(options)
 

data/lib/lono/cfn/opts.rb

@@ -1,11 +1,13 @@
 class Lono::Cfn
   class Opts < Lono::Opts
     def create
+      notification_arns_option
       base_options
       wait_options
     end
 
     def update
+      notification_arns_option
       base_options
       update_options
       wait_options
@@ -76,5 +78,11 @@ class Lono::Cfn
         option :param_preview, type: :boolean, default: true, desc: "Show parameter diff preview."
       end
     end
+
+    def notification_arns_option
+      with_cli_scope do
+        option :notification_arns, type: :array, desc: "The Simple Notification Service (SNS) topic ARNs to publish stack related events."
+      end
+    end
   end
 end

data/lib/lono/cfn/preview/changeset.rb

@@ -31,6 +31,7 @@ module Lono::Cfn::Preview
         stack_name: @stack,
         parameters: parameters,
       }
+      options[:notification_arns] = notification_arns if notification_arns
       options[:tags] = tags unless tags.empty?
       set_template_url!(options)
       show_options(options, "cfn.create_change_set")

data/lib/lono/cfn/update.rb

@@ -46,6 +46,7 @@ class Lono::Cfn
         capabilities: capabilities, # ["CAPABILITY_IAM", "CAPABILITY_NAMED_IAM"]
         disable_rollback: !@options[:rollback],
       }
+      options[:notification_arns] = notification_arns if notification_arns
       options[:tags] = tags unless tags.empty?
       set_template_url!(options)
       show_options(options, "cfn.update_stack")

data/lib/lono/configset.rb

@@ -10,7 +10,7 @@ module Lono
     long_desc Help.text(:generate)
     option :resource, default: "PretendResource", desc: "Set the @resource instance variable availalbe in the configset"
     def generate(configset)
-      Generator.new(options.merge(configset: configset)).run
+      Generator.new(options.merge(configset: configset, cli: true)).run
     end
   end
 end

data/lib/lono/configset/combiner.rb

@@ -56,7 +56,7 @@ class Lono::Configset
 
         data = {
           registry: Lono::Jade::Registry.new(["#{logical_id}OriginalConfigset"], resource: logical_id),
-          metdata_configset: attributes["Metadata"]
+          metdata_configset: {"Metadata" => attributes["Metadata"]} #  # wrap metadata to create right structure
         }
         configsets << data
       end
@@ -72,40 +72,67 @@ class Lono::Configset
       end
 
       metadata_map = {}
+      configsets_map = {}
 
       @sets.each_with_index do |array, i|
         padded_i = "%03d" % i
         registry, metadata = array
-        name, resource = registry.name, registry.resource
 
-        metadata_map[resource] ||= {"AWS::CloudFormation::Init" => {"configSets" => {}}}
-        configSets = metadata_map[resource]["AWS::CloudFormation::Init"]["configSets"]
+        # metadata example (full structure):
+        #
+        #     {"Metadata"=>
+        #       {"AWS::CloudFormation::Init"=>
+        #         {"configSets"=>{"default"=>["aaa1", "aaa2"]},
+        #         "aaa1"=>{"commands"=>{"test"=>{"command"=>"echo from-aaa1 > test1.txt"}}},
+        #         "aaa2"=>
+        #           {"commands"=>{"test"=>{"command"=>"echo from-aaa2 > test1.txt"}}}}}}
 
-        configSets["default"] ||= []
-        configSets["default"] << {"ConfigSet" => name}
+        name, resource = registry.name, registry.resource
+        configsets = configsets_map[resource] ||= {}
 
         validate_structure!(name, metadata)
-        init = metadata["AWS::CloudFormation::Init"]
+
+        new_metadata = metadata["Metadata"].dup
+        init = new_metadata["AWS::CloudFormation::Init"] # important: adjust data by reference
 
         if init.key?("configSets")
+          validate_simple!(registry, new_metadata["AWS::CloudFormation::Init"]["configSets"]) # validate original configset for only simple elements
+
+          # 1. expand each config as its own config, flattening to top-level
           cs = init.delete("configSets") # Only support configSets with simple Array of Strings
-          validate_simple!(registry, cs)
-          configSets[name] = cs["default"].map {|c| "#{padded_i}_#{c}" }
+          new_config_set = {}
+          new_config_set[name] = cs["default"].map {|c| "#{padded_i}_#{c}" }
           init.transform_keys! { |c| "#{padded_i}_#{c}" }
+
+          # Rebuild default configSet, append the new complex ConfigSet structure with each iteration
+          configsets["default"] ||= []
+          configsets["default"] << {"ConfigSet" => name}
+          configsets.merge!(new_config_set) # add each config from #1 to the top-level
+
+          init["configSets"] = configsets # replace new configset
         else # simple config
+          init["configSets"] = configsets # adjust data by reference
+          configsets["default"] ||= []
+          configsets["default"] << {"ConfigSet" => name}
+
+          # build new config
           config_key = "#{padded_i}_single_generated"
-          configSets[name] = [config_key]
-          init = {config_key => init["config"]}
+          configsets[name] = [config_key]
+          new_config = {config_key => init["config"]}
+          # replace old config with new one
+          init.delete("config") # delete original simple config
+          init.merge!(new_config)
         end
 
-        metadata_map[resource]["AWS::CloudFormation::Init"].merge!(init)
+        metadata_map[resource] ||= {"Metadata" => {}}
+        metadata_map[resource]["Metadata"].deep_merge!(new_metadata)
         @map[resource] = metadata_map[resource]
       end
       @map
     end
 
     def validate_structure!(name, metadata)
-      return if metadata.is_a?(Hash) && metadata.key?("AWS::CloudFormation::Init")
+      return if metadata.is_a?(Hash) && metadata.dig("Metadata", "AWS::CloudFormation::Init")
 
       puts "ERROR: The #{name} configset does not appear to have a AWS::CloudFormation::Init key".color(:red)
       puts "Please double check the #{name} configset.yml structure"