lono 6.1.11 → 7.0.0

data/lib/lono/configset/preparer.rb

@@ -0,0 +1,44 @@
+class Lono::Configset
+  class Preparer < Lono::AbstractBase
+    def initialize(options={})
+      super
+      @blueprint = Register::Blueprint.new(options)
+      @project   = Register::Project.new(options)
+      @meta      = Meta.new(options)
+      @resolver  = Resolver.new
+    end
+
+    def run
+      register
+      resolve_dependencies
+      register_dependencies
+      materialize
+      validate_all! # run after final materializer
+    end
+
+    # Stores configsets registry items
+    def register
+      @project.register   # IE: evaluates configs/BLUEPRINT/configsets/base.rb
+      @blueprint.register # IE: evaluates BLUEPRINT/config/configsets.rb
+    end
+
+    def resolve_dependencies
+      jades = Lono::Jade.tracked  # at this point only top-level
+      @resolver.resolve(jades)
+    end
+
+    def register_dependencies
+      @resolver.register
+    end
+
+    def materialize
+      jades = Lono::Jade.downloaded
+      Materializer::Final.new.build(jades)
+    end
+
+    def validate_all!
+      @blueprint.validate!
+      @project.validate!
+    end
+  end
+end

data/lib/lono/configset/register/base.rb

@@ -0,0 +1,121 @@
+# Subclasses must implement:
+#
+#    evaluate
+#
+module Lono::Configset::Register
+  class Base < Lono::AbstractBase
+    class_attribute :configsets
+    class_attribute :validations
+    class_attribute :source
+
+    include Dsl
+    include Lono::Configset::EvaluateFile
+
+    def register
+      evaluate
+      jadify
+    end
+
+    def jadify
+      self.class.configsets.each do |registry|
+        Lono::Jade.new(registry.name, jade_type, registry)
+      end
+    end
+
+    def jade_type
+      finder_class.to_s.sub('Lono::Finder::','').underscore
+    end
+
+    # Used in Base#validate!
+    def finder_class
+      case self
+      when Lono::Configset::Register::Blueprint
+        Lono::Finder::Blueprint::Configset
+      when Lono::Configset::Register::Project
+        Lono::Finder::Configset
+      end
+    end
+
+    # Store to be able to provide the validation errors altogether later.
+    def store_for_validation(registry)
+      # save caller line to use later for pointing to exactly line
+      caller_line = caller.grep(/evaluate_file/).first
+      registry.caller_line = caller_line
+      # huge performance improvement by only validating the first configset registration of duplicate gems
+      names = self.class.validations.map {|r| r.name }
+      self.class.validations << registry unless names.include?(registry.name)
+    end
+
+    # Validate the configset before building templates. So user finds out about errors early.
+    def validate!
+      errors = []
+      self.class.validations.each do |registry|
+        config = finder_class.find(registry.name) # finder_class implemented in subclass
+        errors << [:finder_missing, registry] unless config
+
+        if registry.depends_on.nil? && registry.resource.nil?
+          errors << [:resource_missing, registry]
+        end
+      end
+
+      return if errors.empty? # all good
+      show_errors_and_exit!(errors)
+    end
+
+    def show_errors_and_exit!(errors)
+      errors.each do |error_type, registry|
+        name, caller_line = registry.name, registry.caller_line
+        case error_type
+        when :finder_missing
+          puts "ERROR: Configset with name #{name} not found. Please double check Gemfile and configs/#{@blueprint}/configsets files.".color(:red)
+          pretty_trace(caller_line)
+        when :resource_missing
+          puts "ERROR: Configset with name #{name} does not specify resource. The resource key is required.".color(:red)
+          pretty_trace(caller_line)
+          raise
+        end
+      end
+      exit 1
+    end
+
+    def pretty_trace(caller_line)
+      md = caller_line.match(/(.*\.rb):(\d+):/)
+      path, error_line_number = md[1], md[2].to_i
+
+      context = 5 # lines of context
+      top, bottom = [error_line_number-context-1, 0].max, error_line_number+context-1
+
+      puts "Showing file: #{path}"
+      lines = IO.read(path).split("\n")
+      spacing = lines.size.to_s.size
+      lines[top..bottom].each_with_index do |line_content, index|
+        current_line_number = top+index+1
+        if current_line_number == error_line_number
+          printf("%#{spacing}d %s\n".color(:red), current_line_number, line_content)
+        else
+          printf("%#{spacing}d %s\n", current_line_number, line_content)
+        end
+      end
+    end
+
+  public
+    class << self
+      def clear!
+        self.configsets = []
+        self.validations = []
+      end
+
+      def prepend(registry)
+        self.configsets.unshift(registry) unless has?(registry)
+      end
+
+      def append(registry)
+        self.configsets << registry unless has?(registry)
+      end
+
+      def has?(registry)
+        configsets.detect { |r| r.name == registry.name && r.args == registry.args }
+      end
+    end
+  end
+end

data/lib/lono/configset/register/blueprint.rb

@@ -0,0 +1,16 @@
+module Lono::Configset::Register
+  class Blueprint < Base
+    self.configsets = []
+    self.validations = []
+
+    def evaluate
+      path = find_configsets
+      evaluate_file(path)
+    end
+
+    def find_configsets
+      path = "#{Lono.blueprint_root}/config/configsets.rb"
+      path if File.exist?(path)
+    end
+  end
+end

data/lib/lono/configset/register/dsl.rb

@@ -0,0 +1,15 @@
+module Lono::Configset::Register
+  module Dsl
+    def configset(*args)
+      options = args.last.is_a?(Hash) ? args.pop : {}
+      registry = Lono::Configset::Registry.new(args, options)
+      self.class.append(registry)
+      store_for_validation(registry)
+    end
+
+    # DSL
+    def source(v)
+      self.class.source = v
+    end
+  end
+end

data/lib/lono/configset/register/project.rb

@@ -0,0 +1,12 @@
+module Lono::Configset::Register
+  class Project < Base
+    self.configsets = []
+    self.validations = []
+
+    def evaluate
+      location = Lono::ConfigLocation.new("configsets", @options, Lono.env)
+      evaluate_file(location.lookup_base) if location.lookup_base
+      evaluate_file(location.lookup) if location.lookup # config file
+    end
+  end
+end

data/lib/lono/configset/registry.rb

@@ -0,0 +1,34 @@
+class Lono::Configset
+  class Registry
+    attr_reader :args, :options
+    attr_accessor :caller_line, :parent, :depends_on
+    def initialize(args, options)
+      @args, @options = args, options
+    end
+
+    def name
+      @args.first
+    end
+
+    def resource
+      @options[:resource]
+    end
+
+    def resource=(v)
+      @options[:resource] = v
+    end
+
+    def gem_options
+      options = @options.dup
+      # Delete special options that is not supported by bundler Gemfile
+      options.delete(:repo)
+      options.delete(:resource)
+      options.delete(:vars)
+      options
+    end
+
+    def vars
+      options[:vars] || {}
+    end
+  end
+end

data/lib/lono/configset/resolver.rb

@@ -0,0 +1,42 @@
+class Lono::Configset
+  class Resolver
+    extend Memoist
+
+    @@dependencies = [] # save to later regsiter configsets
+
+    @@resolving_message_shown = false
+    def resolve(*unresolved)
+      unresolved.flatten! # initially only top-level
+      puts "Resolving dependencies..." if !@@resolving_message_shown && !unresolved.empty?
+      puts "Resolving #{unresolved.map(&:name)}" if ENV['LONO_DEBUG_CONFIGSET']
+
+      @@resolving_message_shown = true
+
+      unresolved.each do |jade|
+        jade.check_for_circular_dependency!
+        jade.materialize
+        jade.dependencies.each do |j|
+          @@dependencies << j # store for later registration
+          unless j.resolved? or unresolved.include?(j)
+            resolve(j)
+          end
+        end
+        jade.resolved! # resolve after depth-first tranversal. So all dependencies have also been resolved at this point.
+      end
+    end
+
+    def register
+      @@dependencies.each do |jade|
+        # dependency jades have minimal registry info. For additional info is pulled from stored reference data in
+        # jade like jade.resource_from_parent.
+        registry = jade.registry
+        registry.resource = jade.resource_from_parent
+        if jade.type == "blueprint/configset"
+          Register::Blueprint.prepend(registry)
+        elsif jade.type == "configset"
+          Register::Project.prepend(registry)
+        end
+      end
+    end
+  end
+end

data/lib/lono/conventions.rb

@@ -1,11 +1,22 @@
 module Lono
-  module Conventions
-    # Think can make this a module, but need to figure out how it fits with lono cfn
-    def template_param_convention(options)
-      options = options.deep_symbolize_keys
-      template = options[:template] || @blueprint
-      param = options[:param] || template || @blueprint
-      [template, param]
+  class Conventions
+    attr_reader :stack, :blueprint, :template, :param
+    def initialize(options)
+      @options = options
+      @stack, @blueprint, @template, @param = naming_conventions(options)
+    end
+
+    def naming_conventions(options)
+      o = options.deep_symbolize_keys
+      stack = o[:stack]
+      blueprint = o[:blueprint] || o[:stack]
+      template = o[:template] || blueprint
+      param = o[:param] || template || blueprint
+      [stack, blueprint, template, param]
+    end
+
+    def values
+      [@stack, @blueprint, @template, @param]
     end
   end
 end

data/lib/lono/core.rb

@@ -39,23 +39,6 @@ module Lono
       end
     end
 
-    # Precedence (highest to lowest)
-    #   1. LONO_SUFFIX
-    #   2. .current/lono
-    #   3. config/settings.yml
-    def suffix
-      suffix = ENV['LONO_SUFFIX'] # highest precedence
-      suffix ||= Cfn::Current.suffix
-      unless suffix
-        settings = Setting.new.data
-        suffix ||= settings["stack_name_suffix"] # lowest precedence
-      end
-
-      return if suffix&.empty?
-      suffix
-    end
-    memoize :suffix
-
     # Do not use the Setting#data to load the profile because it can cause an
     # infinite loop then if we decide to use Lono.env from within settings class.
     def settings
@@ -67,9 +50,22 @@ module Lono
     end
     memoize :settings
 
-    def pro_version
-      installed = Gem::Specification.detect { |spec| spec.name == 'lono-pro' }
-      installed ? Lono::Pro::VERSION : "not installed"
+    def lono_pro_removal_check!
+      if lono_pro_installed?
+        puts "ERROR: A lono-pro gem installation has been detected.".color(:red)
+        puts <<~EOL
+          The lono-pro gem is now a part of lono itself. The lono-pro gem has been deprecated.
+          Please uninstall the lono-pro gem and remove it from your Gemfile to continue.
+        EOL
+        exit 1
+      end
+    end
+
+    def lono_pro_installed?
+      Lono::Pro::VERSION
+      true
+    rescue NameError
+      false
     end
 
   private

data/lib/lono/default/settings.yml

@@ -1,13 +1,12 @@
-# More info: http://lono.cloud/docs/settings/
+# More info: https://lono.cloud/docs/settings/
 # The base config is specially treated. It gets included the other environments automatically.
 # Yaml also directly supports merging with & and <<* syntax but doing it automatically
 # for a cleaner syntax.
 base:
-  # http://lono.cloud/docs/app-scripts/
+  # https://lono.cloud/docs/app-scripts/
   # extract_scripts:
   #   to: "/opt"
   #   as: "ec2-user"
-  # stack_name_suffix: random # tack on a 3 char random string at the end of the stack name for lono cfn create
 
 development:
   # When you have AWS_PROFILE set to one of these values, lono will switch to the desired

data/lib/lono/ext/bundler.rb

@@ -0,0 +1,7 @@
+# Bundler 2.0 does yet not have with_unbundled_env
+# Bundler 2.1 deprecates with_clean_env for with_unbundled_env
+
+require "bundler"
+def Bundler.with_unbundled_env(&block)
+  with_clean_env(&block)
+end unless Bundler.respond_to?(:with_unbundled_env)

data/lib/lono/file_uploader.rb

@@ -1,19 +1,22 @@
 module Lono
-  class FileUploader
+  class FileUploader < AbstractBase
     include Lono::Template::AwsService
     extend Memoist
 
-    def initialize(blueprint, options={})
-      @blueprint, @options = blueprint, options
+    def initialize(options={})
+      super
       @checksums = {}
       @prefix = "#{folder_key}/#{Lono.env}/#{blueprint}/files" # s3://s3-bucket/folder/development/files
     end
 
     def upload_all
-      puts "Uploading app/files..."
+      # pattern = "#{Lono.blueprint_root}/app/files/**/*"
+      # size = Dir.glob(pattern).size
+      # puts "size #{size}"
+      # return unless Dir.glob(pattern).size > 0
+      # puts "Uploading app/files2..."
       load_checksums!
 
-      pattern = "#{Lono.blueprint_root}/app/files/**/*"
       Dir.glob(pattern).each do |path|
         next if ::File.directory?(path)
         s3_upload(path)

data/lib/lono/finder/base.rb

@@ -0,0 +1,140 @@
+require "bundler"
+require "text-table"
+
+module Lono::Finder
+  class Base
+    extend Memoist
+
+    def initialize(lono_root: nil, blueprint_root: nil)
+      @lono_root = lono_root || Lono.root
+      @blueprint_root = blueprint_root || Lono.blueprint_root
+    end
+
+    # Returns root path of component: blueprint or configset
+    def find(name, local_only: false)
+      all = find_all(local_only: local_only)
+      all.find { |jadespec| jadespec.name == name }
+    end
+
+    def find_all(local_only: false)
+      if local_only
+        local
+      else
+        local + materialized
+      end
+    end
+
+    # overridden in finder/blueprint/configset.rb
+    def local
+      project + vendor + gems
+    end
+
+    def project
+      roots = path_roots("#{@lono_root}/app/#{type.pluralize}")
+      components(roots, "project")
+    end
+
+    def vendor
+      roots = path_roots("#{@lono_root}/vendor/#{type.pluralize}")
+      components(roots, "vendor")
+    end
+
+    def gems
+      components(gem_roots, "gem")
+    end
+
+    # Folders that each materialized gems to tmp/configsets
+    def materialized
+      components(materialized_gem_roots, "materialized")
+    end
+
+    # Components: blueprints or configsets
+    # Returns array of config Hashes. Example structure:
+    #
+    #     [{
+    #       name: "cfn-hup",
+    #       root: "/path/to/gem/root",
+    #       source_type: "project",
+    #     },...]
+    #
+    def components(roots, source_type)
+      components = []
+      roots.each do |root|
+        next unless detect?(root)
+        jadespec = Lono::Jadespec.new(root, source_type)
+        components << jadespec
+      end
+      components
+    end
+    memoize :components
+
+    def detect?(root)
+      expr = "#{root}/#{detection_path}"
+      Dir.glob(expr).size > 0
+    end
+
+    # Used for blueprints, configsets, and blueprint/configsets
+    def list(options={})
+      table = Text::Table.new
+      table.head = ["Name", "Path", "Type"]
+
+      components = find_all
+      components.each do |jadespec|
+        pretty_path = jadespec.root.sub("#{Lono.root}/", "")
+        unless options[:filter_materialized] && jadespec.source_type == "materialized"
+          table.rows << [jadespec.name, pretty_path, jadespec.source_type]
+        end
+      end
+
+      if table.rows.empty?
+        puts "No #{type.pluralize} found."
+      else
+        puts(options[:message] || "Available #{type.pluralize}:")
+        puts table
+      end
+    end
+
+  private
+    def path_roots(path)
+      Dir.glob("#{path}/*").to_a
+    end
+
+    # Example return:
+    #
+    #     [
+    #       "/home/ec2-user/.rbenv/versions/2.5.6/lib/ruby/gems/2.5.0/gems/rake-13.0.1",
+    #       "/home/ec2-user/.rbenv/versions/2.5.6/lib/ruby/gems/2.5.0/gems/thor-0.20.3"
+    #     ]
+    #
+    def gem_roots
+      Bundler.load.specs.map do |spec|
+        spec.full_gem_path
+      end
+    end
+    memoize :gem_roots
+
+    def materialized_gem_roots
+      gemfile_lock = "#{Lono.root}/tmp/configsets/Gemfile.lock"
+      return [] unless File.exist?(gemfile_lock)
+      parser = Bundler::LockfileParser.new(Bundler.read_file(gemfile_lock))
+      specs = parser.specs
+      # __materialize__ only exists in Gem::LazySpecification and not in Gem::Specification
+      specs.each { |spec| spec.__materialize__ }
+      specs.map do |spec|
+        spec.full_gem_path
+      end
+    end
+    memoize :materialized_gem_roots
+
+  public
+    class << self
+      def find(name)
+        new.find(name)
+      end
+
+      def list(options={})
+        new.list(options)
+      end
+    end
+  end
+end