lono 8.0.0.pre.rc2 → 8.0.0.pre.rc3

data/lib/lono/app_file/build/lambda_layer/ruby_packager.rb

@@ -1,153 +0,0 @@
-class Lono::AppFile::Build::LambdaLayer
-  # Based on jets
-  class RubyPackager
-    include Lono::Utils::Rsync
-
-    def initialize(blueprint, registry_item)
-      @blueprint.name, @registry_item = blueprint, registry_item
-
-      @registry_name = @registry_item.name.sub(/\/$/,'')
-      @app_root = "#{@blueprint.root}/app/files/#{@registry_name}"
-    end
-
-    def build
-      return unless gemfile_exist?
-
-      bundle_install
-      package
-    end
-
-    # We consolidate all gems to opt
-    def package
-      setup_bundle_config(output_area)
-      # copy_cache_gems # TODO: might not need this cache
-      consolidate_gems_to_opt
-    end
-
-    # Also restructure the folder from:
-    #   vendor/gems/ruby/2.5.0
-    # To:
-    #   ruby/gems/2.5.0
-    #
-    # For Lambda Layer structure
-    def consolidate_gems_to_opt
-      src = "#{cache_area}/vendor/gems/ruby/#{ruby_folder}"
-      dest = "#{opt_area}/ruby/gems/#{ruby_folder}"
-      rsync_and_link(src, dest)
-    end
-
-    def rsync_and_link(src, dest)
-      FileUtils.mkdir_p(dest)
-      rsync(src, dest)
-
-      # create symlink in output path not the cache path
-      symlink_dest = "#{output_area}/vendor/gems/ruby/#{ruby_folder}"
-      logger.info "symlink_dest #{symlink_dest}"
-      FileUtils.rm_rf(symlink_dest) # blow away original 2.5.0 folder
-
-      # Create symlink that will point to the gems in the Lambda Layer:
-      #   stage/opt/ruby/gems/2.5.0 -> /opt/ruby/gems/2.5.0
-      FileUtils.mkdir_p(File.dirname(symlink_dest))
-      FileUtils.ln_sf("/opt/ruby/gems/#{ruby_folder}", symlink_dest)
-    end
-
-    def ruby_folder
-      major, minor, _ = RUBY_VERSION.split('.')
-      [major, minor, '0'].join('.') # 2.5.1 => 2.5.0
-    end
-
-    # Installs gems on the current target system: both compiled and non-compiled.
-    # If user is on a macosx machine, macosx gems will be installed.
-    # If user is on a linux machine, linux gems will be installed.
-    #
-    # Copies Gemfile* to /tmp/jets/demo/cache folder and installs
-    # gems with bundle install from there.
-    #
-    # We take the time to copy Gemfile and bundle into a separate directory
-    # because it gets left around to act as a 'cache'.  So, when the builds the
-    # project gets built again not all the gems from get installed from the
-    # beginning.
-    def bundle_install
-      logger.info "Bundling: running bundle install in cache area: #{cache_area}."
-
-      rsync(output_area, cache_area)
-
-      # Uncomment out to always remove the cache/vendor/gems to debug
-      # FileUtils.rm_rf("#{cache_area}/vendor/gems")
-
-      # Remove .bundle folder so .bundle/config doesnt affect how gems are packages
-      # Not using BUNDLE_IGNORE_CONFIG=1 to allow home ~/.bundle/config to affect bundling though.
-      # This is useful if you have private gems sources that require authentication. Example:
-      #
-      #    bundle config gems.myprivatesource.com user:pass
-      #
-
-      FileUtils.rm_rf("#{cache_area}/.bundle")
-      require "bundler" # dynamically require bundler so user can use any bundler
-      setup_bundle_config(cache_area)
-      ::Bundler.with_unbundled_env do
-        sh "cd #{cache_area} && env bundle install"
-      end
-
-      remove_bundled_with("#{cache_area}/Gemfile.lock")
-      # Fixes really tricky bug where Gemfile and Gemfile.lock is out-of-sync. Details: https://gist.github.com/tongueroo/b5b0d0c924a4a1633eee514795e4b04b
-      FileUtils.cp("#{cache_area}/Gemfile.lock", "#{Lono.root}/output/#{@blueprint.name}/files/#{@registry_name}/Gemfile.lock")
-
-      logger.info 'Bundle install success.'
-    end
-
-    # Remove the BUNDLED WITH line since we don't control the bundler gem version on AWS Lambda
-    # And this can cause issues with require 'bundler/setup'
-    def remove_bundled_with(gemfile_lock)
-      lines = IO.readlines(gemfile_lock)
-
-      # amount is the number of lines to remove
-      new_lines, capture, count, amount = [], true, 0, 2
-      lines.each do |l|
-        capture = false if l.include?('BUNDLED WITH')
-        if capture
-          new_lines << l
-        end
-        if capture == false
-          count += 1
-          capture = count > amount # renable capture
-        end
-      end
-
-      content = new_lines.join('')
-      IO.write(gemfile_lock, content)
-    end
-
-    def setup_bundle_config(dir)
-      text =<<-EOL
----
-BUNDLE_PATH: "vendor/gems"
-BUNDLE_WITHOUT: "development:test"
-EOL
-      bundle_config = "#{dir}/.bundle/config"
-      FileUtils.mkdir_p(File.dirname(bundle_config))
-      IO.write(bundle_config, text)
-    end
-
-    def output_area
-      "#{Lono.root}/output/#{@blueprint.name}/files/#{@registry_name}"
-    end
-
-    def build_area
-      "#{Lono.root}/output/#{@blueprint.name}/lambda_layers/#{@registry_name}"
-    end
-
-    def cache_area
-      "#{build_area}/cache"
-    end
-
-    def opt_area
-      "#{build_area}/opt"
-    end
-
-    def gemfile_exist?
-      gemfile_path = "#{@app_root}/Gemfile"
-      File.exist?(gemfile_path)
-    end
-  end
-end

data/lib/lono/app_file/build/lambda_layer.rb

@@ -1,20 +0,0 @@
-class Lono::AppFile::Build
-  class LambdaLayer
-    def initialize(blueprint, registry_item)
-      @blueprint.name, @registry_item = blueprint, registry_item
-    end
-
-    def build
-      lang = @registry_item.options[:lang]
-      unless lang =~ /ruby/
-        logger.info "WARN: Currently only support ruby lambda layers".color(:yellow)
-        return
-      end
-
-      klass_name = "Lono::AppFile::Build::LambdaLayer::#{lang.camelize}Packager"
-      klass = klass_name.constantize
-      packager = klass.new(@blueprint.name, @registry_item)
-      packager.build
-    end
-  end
-end

data/lib/lono/app_file/build.rb

@@ -1,79 +0,0 @@
-require "thor"
-
-module Lono::AppFile
-  class Build < Base
-    include Lono::Utils::Item::Zip
-
-    def initialize_variables
-      @output_files_path = "#{Lono.root}/output/#{@blueprint.name}/files"
-    end
-
-    def run
-      return unless detect_files?
-      logger.info "Building app/files"
-      build_all
-    end
-
-    def build_all
-      clean_output
-      validate_files!
-      copy_to_output
-      build_layers
-      compress_output
-    end
-
-    def validate_files!
-      items = Registry.items + Registry.layers
-      missing = items.select do |item|
-        !File.exist?(item.src_path)
-      end
-      missing_paths = missing.map { |item| item.src_path }.uniq
-      unless missing_paths.empty?
-        logger.info "ERROR: These app/files are missing were used by the s3_key method but are missing".color(:red)
-        missing_paths.each do |path|
-          logger.info "    #{path}"
-        end
-        logger.info "Please double check that they exist."
-        exit 1
-      end
-    end
-
-    def build_layers
-      layer_items = Registry.layers
-      layer_items.each do |item|
-        LambdaLayer.new(@blueprint, item).build
-      end
-    end
-
-    def compress_output
-      Registry.items.each do |item|
-        # type can be lambda_layer or file
-        if item.type == "lambda_layer" || item.exist?
-          zip(item)
-        else
-          logger.info "WARN: #{item.src_path} does not exist. Double check that the path is correct in the s3_key call.".color(:yellow)
-        end
-      end
-    end
-
-    def copy_to_output
-      override_source_paths("#{@blueprint.root}/app/files")
-      self.destination_root = @output_files_path
-      directory(".", verbose: false, context: template_context.get_binding) # Thor::Action
-    end
-
-    def clean_output
-      FileUtils.rm_rf(@output_files_path)
-    end
-
-    def detect_files?
-      app_files = Dir["#{@blueprint.root}/app/files/*"]
-      if app_files.empty?
-        false
-      else
-        logger.info "Detected app/files"
-        true
-      end
-    end
-  end
-end

data/lib/lono/app_file/registry/item.rb

@@ -1,24 +0,0 @@
-class Lono::AppFile::Registry
-  # Holds metadata about the item in the regsitry.
-  class Item
-    include Lono::Utils::Item::FileMethods
-
-    attr_reader :name, :options, :type
-    def initialize(name, blueprint, options={})
-      @name, @blueprint.name, @options = name, blueprint, options
-      @type = options[:type] || "file"
-    end
-
-    def src_path
-      "#{@blueprint.root}/app/files/#{@name}"
-    end
-
-    def output_path
-      if @type == "file"
-        "#{Lono.root}/output/#{@blueprint.name}/files/#{@name}"
-      else
-        "#{Lono.root}/output/#{@blueprint.name}/lambda_layers/#{@name}/opt"
-      end
-    end
-  end
-end

data/lib/lono/app_file/registry.rb

@@ -1,16 +0,0 @@
-module Lono::AppFile
-  class Registry
-    cattr_reader :items
-    @@items = []
-
-    class << self
-      def register(name, blueprint, options={})
-        @@items << Item.new(name, blueprint, options) unless @@items.detect { |i| i.name == name && i.type == options[:type] }
-      end
-
-      def layers
-        @@items.select { |i| i.type == "lambda_layer" }
-      end
-    end
-  end
-end

data/lib/lono/app_file/upload.rb

@@ -1,12 +0,0 @@
-module Lono::AppFile
-  class Upload < Base
-    def upload
-      return unless Registry.items.size > 0
-      logger.info "Uploading app/files..."
-
-      Registry.items.each do |item|
-        Lono::S3::Uploader.new(item.zip_file_path).upload
-      end
-    end
-  end
-end

data/lib/lono/builder/context/generic.rb

@@ -1,11 +0,0 @@
-module Lono::Builder::Context
-  class Generic < Lono::CLI::Base
-    include Lono::Builder::Dsl::Syntax
-
-    # For Lono::AppFile::Build usage of Thor::Action directory
-    # For some reason a send(:binding) doesnt work but get_binding like this works.
-    def get_binding
-      binding
-    end
-  end
-end

data/lib/lono/builder/context/loaders/load_files.rb

@@ -1,23 +0,0 @@
-module Lono::Builder::Context::Loaders
-  module LoadFiles
-    # Load custom helper methods from project
-    def load_files(dir)
-      paths = Dir.glob("#{dir}/**/*.rb")
-      paths.sort_by! { |p| p.size } # so namespaces are loaded first
-      paths.each do |path|
-        # helpers = blueprint or project extensions
-        filename = path.sub(%r{.*/helpers/},'').sub('.rb','')
-        module_name = filename.camelize
-
-        # Prepend a period so require works LONO_ROOT is set to a relative path without a period.
-        #
-        # Example: LONO_ROOT=tmp/lono_project
-        first_char = path[0..0]
-        path = "./#{path}" unless %w[. /].include?(first_char)
-
-        require path
-        self.class.send :include, module_name.constantize
-      end
-    end
-  end
-end

data/lib/lono/builder/context/loaders.rb

@@ -1,35 +0,0 @@
-module Lono::Builder::Context
-  module Loaders
-    include LoadFiles
-
-    # Variables in base.rb are overridden by their environment specific variables
-    # file.  Example, file LONO_ENV=dev:
-    #
-    #   config/vars/base.rb
-    #   config/vars/dev.rb - will override any variables in base.rb
-    #   config/vars/base.rb
-    #   config/vars/dev.rb - will override any variables in base.rb
-    #
-    def load_variables
-      layers = Lono::Layering::Layer.new(@blueprint, "vars").paths
-      layers.each do |layer|
-        evaluate_variables_file(layer)
-      end
-    end
-
-    # Load the variables defined in config/vars/* to make available in lono scope.
-    #
-    # NOTE: Was only able to make instance variables avaialble with instance_eval, wasnt able to make local variables
-    # available.
-    def evaluate_variables_file(path)
-      return unless File.exist?(path)
-      instance_eval(IO.read(path), path)
-    end
-
-    # Load blueprint helpers
-    # blueprint helpers override extension helpers
-    def load_blueprint_helpers
-      load_files("#{@blueprint.root}/#{Lono.config.paths.helpers}")
-    end
-  end
-end

data/lib/lono/builder/context/params.rb

@@ -1,6 +0,0 @@
-module Lono::Builder::Context
-  class Params < Generic
-    # Overriding output resource DSL method
-    alias_method :output, :stack_output
-  end
-end

data/lib/lono/builder/context/template.rb

@@ -1,4 +0,0 @@
-module Lono::Builder::Context
-  class Template < Generic
-  end
-end

data/lib/lono/configset/builder.rb

@@ -1,59 +0,0 @@
-module Lono::Configset
-  class Builder
-    include Lono::Utils::Logging
-
-    def initialize(options)
-      @options = options
-      @configset = options[:configset]
-      @type = options[:type] || "project"
-    end
-
-    def run
-      check_configset_exist!
-      structure = build
-      logger.info YAML.dump(structure)
-    end
-
-    def check_configset_exist!
-      exist = !!Lono::Finder::Configset.find(@configset)
-      unless exist
-        logger.info "configset #{@configset.color(:green)} not found."
-        exit 1
-      end
-    end
-
-    def build
-      # Examples:
-      #   Erb.new(options).build
-      #   Dsl.new(options).build
-      builder_class = "Lono::Configset::Strategy::#{strategy.camelize}"
-      builder_class = Object.const_get(builder_class)
-      full = builder_class.new(@options.merge(root: configset_root)).build
-      if @options[:cli]
-        full["Metadata"] # contains AWS::CloudFormation::Init and optional AWS::CloudFormation::Authentication
-      else
-        full # Combiner uses full metadata structure
-      end
-    end
-
-    def strategy
-      jadespec = Lono::Jadespec.new(configset_root, "unknown") # abusing Jadespec to get strategy
-      jadespec.lono_strategy
-    end
-
-    def configset_root
-      finder = finder_class.new
-      found = finder.find(@configset, local_only: false)
-      found.root if found
-    end
-
-    def finder_class
-      case @type
-      when "project"
-        Lono::Finder::Configset
-      when "blueprint"
-        Lono::Finder::Blueprint::Configset
-      end
-    end
-  end
-end

data/lib/lono/configset/combiner.rb

@@ -1,164 +0,0 @@
-require "yaml"
-
-module Lono::Configset
-  class Combiner
-    include Lono::Utils::Logging
-
-    def initialize(cfn, options={})
-      @cfn, @options = cfn, options
-
-      @sets = []
-      @map = {} # stores resource logical id => metadata cfn-init
-    end
-
-    def generator_options(registry, options={})
-      o = @options.merge(configset: registry.name, resource: registry.resource)
-      o.merge(options)
-    end
-
-    def metadata_map
-      return {} unless additional_configsets?
-
-      existing_configsets.each do |data|
-        add(data[:registry], data[:metdata_configset])
-      end
-
-      Register::Blueprint.configsets.each do |registry|
-        generator = Lono::Configset::Generator.new(generator_options(registry, type: "blueprint"))
-        cloudformation_init = generator.build
-        add(registry, cloudformation_init)
-      end
-      Register::Project.configsets.each do |registry|
-        generator = Lono::Configset::Generator.new(generator_options(registry, type: "project"))
-        cloudformation_init = generator.build
-        add(registry, cloudformation_init)
-      end
-
-      combine
-      Register::Blueprint.clear! # in case of lono build for all templates
-      Register::Project.clear! # in case of lono build for all templates
-      @map
-    end
-
-    def add(registry, metadata)
-      @sets << [registry, metadata.dup]
-    end
-
-    def additional_configsets?
-      !Register::Blueprint.configsets.empty? || !Register::Project.configsets.empty?
-    end
-
-    # Normalized/convert cfn template to mimic the registry format
-    def existing_configsets
-      configsets = []
-      @cfn["Resources"].each do |logical_id, attributes|
-        init = attributes.dig("Metadata", "AWS::CloudFormation::Init")
-
-        next unless init
-
-        data = {
-          registry: Lono::Jade::Registry.new(["#{logical_id}OriginalConfigset"], resource: logical_id),
-          metdata_configset: {"Metadata" => attributes["Metadata"]} #  # wrap metadata to create right structure
-        }
-        configsets << data
-      end
-      configsets
-    end
-
-    def combine
-      # Remove duplicate configsets. Can happen if same configset is in blueprint and project.
-      # Ugly because of the sets structure.
-      @sets.uniq! do |array|
-        registry, _ = array
-        registry.name
-      end
-
-      metadata_map = {}
-      configsets_map = {}
-
-      @sets.each_with_index do |array, i|
-        padded_i = "%03d" % i
-        registry, metadata = array
-
-        # metadata example (full structure):
-        #
-        #     {"Metadata"=>
-        #       {"AWS::CloudFormation::Init"=>
-        #         {"configSets"=>{"default"=>["aaa1", "aaa2"]},
-        #         "aaa1"=>{"commands"=>{"test"=>{"command"=>"echo from-aaa1 > test1.txt"}}},
-        #         "aaa2"=>
-        #           {"commands"=>{"test"=>{"command"=>"echo from-aaa2 > test1.txt"}}}}}}
-
-        name, resource = registry.name, registry.resource
-        configsets = configsets_map[resource] ||= {}
-
-        validate_structure!(name, metadata)
-
-        new_metadata = metadata["Metadata"].dup
-        init = new_metadata["AWS::CloudFormation::Init"] # important: adjust data by reference
-
-        if init.key?("configSets")
-          validate_simple!(registry, new_metadata["AWS::CloudFormation::Init"]["configSets"]) # validate original configset for only simple elements
-
-          # 1. expand each config as its own config, flattening to top-level
-          cs = init.delete("configSets") # Only support configSets with simple Array of Strings
-          new_config_set = {}
-          new_config_set[name] = cs["default"].map {|c| "#{padded_i}_#{c}" }
-          init.transform_keys! { |c| "#{padded_i}_#{c}" }
-
-          # Rebuild default configSet, append the new complex ConfigSet structure with each iteration
-          configsets["default"] ||= []
-          configsets["default"] << {"ConfigSet" => name}
-          configsets.merge!(new_config_set) # add each config from #1 to the top-level
-
-          init["configSets"] = configsets # replace new configset
-        else # simple config
-          init["configSets"] = configsets # adjust data by reference
-          configsets["default"] ||= []
-          configsets["default"] << {"ConfigSet" => name}
-
-          # build new config
-          config_key = "#{padded_i}_single_generated"
-          configsets[name] = [config_key]
-          new_config = {config_key => init["config"]}
-          # replace old config with new one
-          init.delete("config") # delete original simple config
-          init.merge!(new_config)
-        end
-
-        metadata_map[resource] ||= {"Metadata" => {}}
-        metadata_map[resource]["Metadata"].deep_merge!(new_metadata)
-        @map[resource] = metadata_map[resource]
-      end
-      @map
-    end
-
-    def validate_structure!(name, metadata)
-      return if metadata.is_a?(Hash) && metadata.dig("Metadata", "AWS::CloudFormation::Init")
-
-      logger.info "ERROR: The #{name} configset does not appear to have a AWS::CloudFormation::Init key".color(:red)
-      logger.info "Please double check the #{name} configset.yml structure"
-      exit 1
-    end
-
-    def validate_simple!(registry, cs)
-      has_complex_type = cs["default"].detect { |s| !s.is_a?(String) }
-      if has_complex_type
-        message =<<~EOL
-          ERROR: The configset #{registry.name} has a configSets property with a complex type.
-          configSets:
-
-              #{cs}
-
-          lono configsets only supports combining configSets with an Array of Strings.
-        EOL
-        if ENV['LONO_TEST']
-          raise message
-        else
-          logger.info message.color(:red)
-          exit 1
-        end
-      end
-    end
-  end
-end

data/lib/lono/configset/evaluate_file.rb

@@ -1,8 +0,0 @@
-module Lono::Configset
-  module EvaluateFile
-    def evaluate_file(path)
-      return unless path && File.exist?(path)
-      instance_eval(IO.read(path), path)
-    end
-  end
-end

data/lib/lono/configset/meta/dsl.rb

@@ -1,12 +0,0 @@
-class Lono::Configset::Meta
-  module Dsl
-    def depends_on(*args)
-      options = args.last.is_a?(Hash) ? args.pop : {}
-      registry = Lono::Jade::Registry.new(args, options)
-      registry.depends_on = args.first
-      registry.parent = @jade
-      already_has = @jade.depends_ons.detect { |d| d.name == registry.name && d.args == registry.args }
-      @jade.depends_ons << registry unless already_has
-    end
-  end
-end

data/lib/lono/configset/meta.rb

@@ -1,19 +0,0 @@
-module Lono::Configset
-  class Meta
-    extend Memoist
-    include EvaluateFile
-    include Dsl
-
-    class_attribute :registries
-    self.registries = []
-
-    def initialize(jade)
-      @jade = jade
-    end
-
-    def evaluate
-      path = "#{@jade.root}/lib/meta.rb"
-      evaluate_file(path) # DSL depends_on sets @jade.depends_on decorated options
-    end
-  end
-end