lono 7.3.2 → 7.4.0

data/lib/lono/configset/generator.rb

@@ -26,7 +26,12 @@ class Lono::Configset
       #   Dsl.new(options).build
       generator_class = "Lono::Configset::Strategy::#{strategy.camelize}"
       generator_class = Object.const_get(generator_class)
-      generator_class.new(@options.merge(root: configset_root)).build
+      full = generator_class.new(@options.merge(root: configset_root)).build
+      if @options[:cli]
+        full["Metadata"] # contains AWS::CloudFormation::Init and optional AWS::CloudFormation::Authentication
+      else
+        full # Combiner uses full metadata structure
+      end
     end
 
     def strategy

data/lib/lono/configset/list.rb

@@ -41,7 +41,7 @@ class Lono::Configset
       table = Text::Table.new
       table.head = ["Name", "Path", "Type", "From"]
       @final.each do |spec|
-        pretty_root = spec.root.sub("#{Lono.root}/",'')
+        pretty_root = spec.root.sub("#{Lono.root}/",'').sub(ENV["HOME"], "~")
         table.rows << [spec.name, pretty_root, spec.source_type, spec.from]
       end
 

data/lib/lono/configset/register/dsl.rb

@@ -7,8 +7,8 @@ module Lono::Configset::Register
     end
 
     # DSL
-    def source(v)
-      self.class.source = v
-    end
+    # def source(v)
+    #   self.class.source = v
+    # end
   end
 end

data/lib/lono/configset/resolver.rb

@@ -32,11 +32,15 @@ class Lono::Configset
         # jade like jade.resource_from_parent.
         registry = jade.registry
         registry.resource = jade.resource_from_parent
-        if jade.type == "blueprint/configset"
-          Register::Blueprint.prepend(registry)
-        elsif jade.type == "configset"
-          Register::Project.prepend(registry)
-        end
+
+        # Always prepend to Register::Blueprint.configsets to ensure they are run before any other configset.
+        # Leaving old code in place for now. Possibly revisit this approach.
+        Register::Blueprint.prepend(registry)
+        # if jade.type == "blueprint/configset"
+        #   Register::Blueprint.prepend(registry)
+        # elsif jade.type == "configset"
+        #   Register::Project.prepend(registry)
+        # end
       end
     end
   end

data/lib/lono/configset/s3_file/build.rb

@@ -0,0 +1,33 @@
+module Lono::Configset::S3File
+  class Build < Lono::AbstractBase
+    include Lono::Utils::Rsync
+    include Lono::Utils::Item::Zip
+
+    def run
+      Lono::Configset::S3File::Registry.items.each do |item|
+        build(item)
+      end
+    end
+
+    def build(item)
+      copy_to_output(item)
+      compress_output
+    end
+
+    def copy_to_output(item)
+      src = "#{item.root}/lib/files/#{item.name}"
+      dest = "#{Lono.config.output_path}/#{@blueprint}/configsets/#{item.configset}/files/#{item.name}"
+      rsync(src, dest)
+    end
+
+    def compress_output
+      Registry.items.each do |item|
+        if item.exist?
+          zip(item)
+        else
+          puts "WARN: #{item.src_path} does not exist. Double check that the path is correct in the s3_key call.".color(:yellow)
+        end
+      end
+    end
+  end
+end

data/lib/lono/configset/s3_file/item.rb

@@ -0,0 +1,30 @@
+module Lono::Configset::S3File
+  # Holds metadata about the item in the regsitry.
+  class Item
+    include Lono::Utils::Item::FileMethods
+
+    attr_reader :name, :configset, :root
+    def initialize(name, options={})
+      @name, @options = name, options
+      @blueprint = options[:blueprint]
+      @configset = options[:configset]
+      @root = options[:root]
+      @type = options[:type] || 'file'
+    end
+
+    def src_path
+      "#{@root}/lib/files"
+    end
+
+    def output_path
+      "#{Lono.config.output_path}/#{@blueprint}/configsets/#{@configset}/files/#{@name}"
+    end
+
+    def replacement_value
+      aws_data = AwsData.new
+      # "https://s3.amazonaws.com/#{Lono::S3::Bucket.name}/#{s3_path}"
+      # "https://lono-bucket-12di8xz5sy72z.s3-us-west-2.amazonaws.com/stuff/s3-antivirus.tgz"
+      "https://#{Lono::S3::Bucket.name}.s3-#{aws_data.region}.amazonaws.com/#{s3_path}"
+    end
+  end
+end

data/lib/lono/configset/s3_file/registry.rb

@@ -0,0 +1,12 @@
+module Lono::Configset::S3File
+  class Registry
+    cattr_reader :items
+    @@items = []
+
+    class << self
+      def register(name, options)
+        @@items << Item.new(name, options) unless @@items.detect { |i| i.name == name }
+      end
+    end
+  end
+end

data/lib/lono/configset/s3_file/upload.rb

@@ -0,0 +1,12 @@
+module Lono::Configset::S3File
+  class Upload < Lono::AbstractBase
+    def upload
+      return unless Registry.items.size > 0
+      puts "Uploading configset files..."
+
+      Registry.items.each do |item|
+        Lono::S3::Uploader.new(item.zip_file_path).upload
+      end
+    end
+  end
+end

data/lib/lono/configset/strategy/base.rb

@@ -13,6 +13,18 @@ module Lono::Configset::Strategy
     extend Memoist
     include Lono::Configset::EvaluateFile
 
+    # All Lono DSL Helpers and Fn - so configsets have access to instrinic functions like ref
+    # Not including the Lono::Template::Strategy::Dsl::Builder::Syntax since dont need those methods
+    #
+    # Interesting note: must include these modules here so load_project_predefined_variables works.
+    # Since load_project_predefined_variables calls evaluate_file / instance eval which seems to only be
+    # able to access methods from the direct class, not inherited classes like Dsl.
+    #
+    # This allows methods like ref and sub to work in variables files.
+    #
+    include Lono::Template::Strategy::Dsl::Builder::Helpers
+    include Lono::Template::Strategy::Dsl::Builder::Fn
+
     def initialize(options={})
       @options = options
       @configset = options[:configset]
@@ -25,10 +37,18 @@ module Lono::Configset::Strategy
       @evaluation_path = find_evaluation_path # implemented by subclass
       copy_instance_variables
       load_configset_helpers
-      load # implemented by subclass
+      init = load # implemented by subclass
+      finish_full_structure(init)
     end
     memoize :build
 
+    def finish_full_structure(init)
+      full = {"Metadata" => {}}
+      full["Metadata"]["AWS::CloudFormation::Init"] = init["AWS::CloudFormation::Init"]
+      full["Metadata"]["AWS::CloudFormation::Authentication"] = authentication if authentication # only on dsl
+      full.deep_stringify_keys!
+    end
+
     def copy_instance_variables
       load_blueprint_predefined_variables
       load_project_predefined_variables

data/lib/lono/configset/strategy/dsl.rb

@@ -1,13 +1,16 @@
 module Lono::Configset::Strategy
   class Dsl < Base
-    include Helpers
-    include Syntax
+    # Configset helpers take higher precedence than Lono DSL Helpers. IE: s3_key is overwritten but s3_bucket is the same
+    include Helpers::Dsl
 
     def initialize(options={})
       super
-      @structure = {} # holds in memory the configset hash structure
+      @structure = {} # holds in memory the configset hash structure to build AWS::CloudFormation::Init
       @current = "main" # current configset
+      @command_counts = Hash.new(0)
       @tracked = []
+      # Also support ability to add AWS::CloudFormation::Authentication
+      @authentication = nil # holds IAM policy info to build AWS::CloudFormation::Authentication
     end
 
     def find_evaluation_path
@@ -18,9 +21,9 @@ module Lono::Configset::Strategy
       evaluate_file(@evaluation_path)
       configsets = @configsets || @tracked.uniq
       configsets = ["main"] if configsets.empty?
-      init = {"configSets" => {"default" => configsets}}.merge(@structure)
-      full_structure = { "AWS::CloudFormation::Init" => init }
-      full_structure.deep_stringify_keys!
+      configsets_structure = {"configSets" => {"default" => configsets}}.merge(@structure)
+      full = { "AWS::CloudFormation::Init" => configsets_structure }
+      full.deep_stringify_keys!
     end
   end
 end

data/lib/lono/configset/strategy/erb.rb

@@ -2,7 +2,7 @@ require "json"
 
 module Lono::Configset::Strategy
   class Erb < Base
-    include Helpers
+    include Helpers::Erb
 
     def find_evaluation_path
       paths = %w[configset.yml configset.json].map { |p| "#{@root}/lib/#{p}" }
@@ -11,6 +11,7 @@ module Lono::Configset::Strategy
 
     def load
       content = RenderMePretty.result(@evaluation_path, context: self)
+      # init structure
       if File.extname(@evaluation_path) == ".yml"
         load_yaml(content)
       else
@@ -18,6 +19,10 @@ module Lono::Configset::Strategy
       end
     end
 
+    def authentication
+      # noop
+    end
+
     def load_yaml(content)
       # Write to file so can use Yamler::Validator
       path = "/tmp/lono/configset.yml"

data/lib/lono/configset/strategy/helpers/dsl.rb

@@ -0,0 +1,8 @@
+module Lono::Configset::Strategy::Helpers
+  module Dsl
+    include Auth
+    include Core
+    include Package
+    include Syntax
+  end
+end

data/lib/lono/configset/strategy/helpers/dsl/auth.rb

@@ -0,0 +1,37 @@
+module Lono::Configset::Strategy::Helpers::Dsl
+  module Auth
+    def authentication(data=nil, force: false)
+      if data.nil?
+        authentication_reader
+      else
+        authentication_setter(data, force)
+      end
+    end
+
+    # data can be either:
+    #
+    #     1. logical id - String
+    #     2. Full AWS::CloudFormation::Authentication value structure
+    #
+    def authentication_reader
+      # AWS::CloudFormation::Authentication
+      case @authentication
+      when String
+        logical_id = @authentication
+        {
+          rolebased: {
+            type: "S3",
+            buckets: [lono_bucket_name],
+            roleName: {Ref: logical_id}, # currently ref meth is not available
+          }
+        }
+      when Hash
+        @authentication
+      end
+    end
+
+    def authentication_setter(data, force=false)
+      @authentication = data unless @authentication || force
+    end
+  end
+end

data/lib/lono/configset/strategy/{dsl/helpers → helpers/dsl}/core.rb

@@ -1,4 +1,4 @@
-module Lono::Configset::Strategy::Dsl::Helpers
+module Lono::Configset::Strategy::Helpers::Dsl
   module Core
     def content_file(path)
       content_path = "#{@root}/lib/content"
@@ -9,5 +9,10 @@ module Lono::Configset::Strategy::Dsl::Helpers
         "File not found: #{file}"
       end
     end
+
+    def s3_key(name)
+      Lono::Configset::S3File::Registry.register(name, blueprint: @blueprint, configset: @configset, root: @root)
+      "file://configset/#{@configset}/#{name}"
+    end
   end
 end

data/lib/lono/configset/strategy/helpers/dsl/package.rb

@@ -0,0 +1,40 @@
+module Lono::Configset::Strategy::Helpers::Dsl
+  module Package
+    # In recent versions of the AmazonLinux2 cfnbootstrap, , the package command doesnt work with updated versions of rubygems.
+    # Get this error
+    #
+    #    invalid option: --no-ri
+    #
+    # More details: https://gist.github.com/tongueroo/569878afdc7eb904490b9ee8b03f304f
+    #
+    # Found the cfnbootstrap version by looking at the source on 2020-03-21 in
+    #
+    #     $ cat /usr/lib/python2.7/site-packages/cfnbootstrap/public_constants.py
+    #     _release = '31'
+    #     _version = '1.4-' + _release
+    #
+    # There is no way to get the version from the /opt/aws/bin/cfn-init command.
+    #
+    # We work around this be using the command instruction and use the gem install and list commands.
+    #
+    #    $ gem list tilt -e -i -v 1.4.0
+    #    false # also $? is 1
+    #    $ gem list tilt -e -i -v 1.4.1
+    #    true # also $? is 0
+    #    $
+    #
+    def gem_package(name, version=nil)
+      unless_clause = "gem list #{name} -e -i "
+      unless_clause += "-v #{version}" if version
+      command("#{name}-gem-install",
+        command: "gem install #{name} #{version}",
+        unless: unless_clause
+      )
+    end
+
+    def yum_package(name, version=nil)
+      versions = [version].compact
+      package("yum", name => versions)
+    end
+  end
+end

data/lib/lono/configset/strategy/helpers/dsl/syntax.rb

@@ -0,0 +1,74 @@
+module Lono::Configset::Strategy::Helpers::Dsl
+  module Syntax
+    extend Memoist
+
+    %w[package group user file service].each do |meth|
+      section = meth.pluralize
+      define_method(meth) do |k, props={}|
+        init_empty(@current, section)
+        current_structure(@current)[section].deep_merge!(k => props)
+      end
+    end
+
+    # Add extra conveniences to command method
+    def command(key, props={})
+      init_empty(@current, "commands")
+
+      # order commands automatically
+      if key !~ /^\d+_/
+        c = @command_counts[@current] += 1 # IE: @command_counts["main"]
+        padded_c = "%03d" % c
+        key = "#{padded_c}_#{key}"
+      end
+
+      # if syntax support
+      if props.key?(:if)
+        if_clause = props.delete(:if)
+        props[:test] = "if #{if_clause} ; then true ; else false ; fi"
+        # returns true  - will run command
+        # returns false - will not run command
+      end
+
+      # unless syntax support
+      if props.key?(:unless)
+        unless_clause = props.delete(:unless)
+        props[:test] = "if #{unless_clause} ; then false ; else true ; fi"
+        # returns true  - will run command
+        # returns false - will not run command
+      end
+
+      current_structure(@current)["commands"].deep_merge!(key => props)
+    end
+
+    # Source has a different signature than the other native methods
+    def source(*args)
+      if args.first.is_a?(Hash)
+        item = args.first
+      else # 2 args form: first element is k, second is
+        k, v, _ = args
+        item = {k => v}
+      end
+
+      init_empty(@current, "sources")
+      current_structure(@current)["sources"].deep_merge!(item)
+    end
+
+    def configset(current)
+      @tracked << current
+      previous, @current = @current, current
+      yield
+      @current = previous
+    end
+
+  private
+    def current_structure(configset)
+      @structure[configset] ||= {}
+    end
+    memoize :current_structure
+
+    def init_empty(configset, section)
+      current = current_structure(configset)
+      current[section] ||= {}
+    end
+  end
+end

data/lib/lono/configset/strategy/{erb/helpers.rb → helpers/erb.rb}

@@ -1,5 +1,5 @@
-class Lono::Configset::Strategy::Erb
-  module Helpers
+module Lono::Configset::Strategy::Helpers
+  module Erb
     def indent(text, indentation_amount)
       text.split("\n").map do |line|
         " " * indentation_amount + line

data/lib/lono/extension/list.rb

@@ -26,7 +26,8 @@ class Lono::Extension
       presenter.header = %w[Name Path Type]
       jadespecs.each do |j|
         if tracked_extension_names.include?(j.name)
-          presenter.rows << [j.name, j.root, j.source_type]
+          pretty_path = j.root.sub("#{Lono.root}/",'').sub(ENV["HOME"], "~")
+          presenter.rows << [j.name, pretty_path, j.source_type]
         end
       end
       presenter.show