lono 7.3.2 → 7.4.0

data/lib/lono/template/strategy/dsl/builder/helpers.rb

@@ -3,6 +3,7 @@ class Lono::Template::Strategy::Dsl::Builder
   module Helpers
     extend Memoist
     include CoreHelper
+    include Ec2Helper
     include FileHelper
     include LookupHelper
     include S3Helper

data/lib/lono/template/strategy/dsl/builder/helpers/ec2_helper.rb

@@ -0,0 +1,27 @@
+module Lono::Template::Strategy::Dsl::Builder::Helpers
+  module Ec2Helper
+    extend Memoist
+
+    def default_vpc
+      resp = ec2.describe_vpcs(filters: [name: "isDefault", values: ["true"]])
+      vpc = resp.vpcs.first
+      vpc ? vpc.vpc_id : "no default vpc found"
+    end
+    memoize :default_vpc
+
+    def default_subnets
+      return "no default subnets because no default vpc found" if default_vpc == "no default vpc found"
+      resp = ec2.describe_subnets(filters: [name: "vpc-id", values: [default_vpc]])
+      subnets = resp.subnets
+      subnets.map(&:subnet_id)
+    end
+    memoize :default_subnets
+
+    def key_pairs(regexp=nil)
+      resp = ec2.describe_key_pairs
+      key_names = resp.key_pairs.map(&:key_name)
+      key_names.select! { |k| k =~ regexp }
+      key_names
+    end
+  end
+end

data/lib/lono/template/strategy/dsl/builder/helpers/s3_helper.rb

@@ -3,6 +3,7 @@ module Lono::Template::Strategy::Dsl::Builder::Helpers
     def s3_bucket
       Lono::S3::Bucket.name
     end
+    alias_method :lono_bucket_name, :s3_bucket
 
     def s3_key(name, options={})
       default = {type: "file"}

data/lib/lono/template/upload.rb

@@ -5,8 +5,6 @@ require 'digest'
 
 class Lono::Template
   class Upload < Lono::AbstractBase
-    include Lono::AwsServices
-
     def initialize(options={})
       super
       @checksums = {}
@@ -14,104 +12,12 @@ class Lono::Template
     end
 
     def run
-      load_checksums!
-
-      say "Uploading CloudFormation templates..."
+      puts "Uploading CloudFormation templates..."
       paths = Dir.glob("#{Lono.config.output_path}/#{@blueprint}/templates/**/*")
       paths.select { |p| File.file?(p) }.each do |path|
-        upload(path)
-      end
-      say "Templates uploaded to s3."
-    end
-
-    # Read existing files on s3 to grab their md5 checksum.
-    # We do this so we can see if we should avoid re-uploading the s3 child template
-    # entirely. If we upload a new child template that does not change AWS CloudFormation
-    # is not smart enough to know that it not has changed. I think all AWS CloudFormation
-    # does is check if the file's timestamp.
-    #
-    # Thought this would result in better AWS Change Set info but AWS still reports child
-    # stacks being changed even though they should not be reported.  Leaving this s3 checksum
-    # in for now.
-    def load_checksums!
-      return if @options[:noop]
-
-      resp = s3.list_objects(bucket: s3_bucket, prefix: @prefix)
-      resp.contents.each do |object|
-        @checksums[object.key] = strip_surrounding_quotes(object.etag)
-      end
-      @checksums
-    end
-
-    def strip_surrounding_quotes(string)
-      string.sub(/^"/,'').sub(/"$/,'')
-    end
-
-    def upload(path)
-      return if @options[:noop]
-
-      path = path.sub("#{Lono.root}/",'')
-      pretty_path = path.sub(/^\.\//, '')
-      key = "#{@prefix}/#{pretty_path.sub(%r{output/templates/},'')}"
-      s3_full_path = "s3://#{s3_bucket}/#{key}"
-
-      local_checksum = Digest::MD5.hexdigest(IO.read(path))
-      remote_checksum = remote_checksum(path)
-      if local_checksum == remote_checksum
-        say("Not modified: #{pretty_path} to #{s3_full_path}".color(:yellow)) unless @options[:noop]
-        return # do not upload unless the checksum has changed
+        Lono::S3::Uploader.new(path).upload
       end
-
-      resp = s3.put_object(
-        body: IO.read(path),
-        bucket: s3_bucket,
-        key: key,
-        storage_class: "REDUCED_REDUNDANCY"
-      ) unless @options[:noop]
-
-      # Example output:
-      # Uploaded: output/templates/docker.yml to s3://boltops-dev/s3_folder/templates/development/docker.yml
-      # Uploaded: output/templates/ecs/private.yml to s3://boltops-dev/s3_folder/templates/development/ecs/private.yml
-      message = "Uploaded: #{pretty_path} to #{s3_full_path}".color(:green)
-      message = "NOOP: #{message}" if @options[:noop]
-      say message
-    end
-
-    # @checksums map has a key format: s3_folder/templates/development/docker.yml
-    #
-    # path = ./output/templates/docker.yml
-    # s3_folder = s3://boltops-dev/s3_folder/templates/development/docker.yml
-    def remote_checksum(path)
-      # first convert the local path to the path format that is stored in @checksums keys
-      # ./output/templates/docker.yml => s3_folder/templates/development/docker.yml
-      pretty_path = path.sub(/^\.\//, '')
-      key = "#{@prefix}/#{pretty_path.sub(%r{output/templates/},'')}"
-      @checksums[key]
-    end
-
-    # https://s3.amazonaws.com/mybucket/s3_folder/templates/production/parent.yml
-    def s3_https_url(template_path)
-      "https://s3.amazonaws.com/#{s3_bucket}/#{@prefix}/#{template_path}"
-    end
-
-    # used for cfn/base.rb def set_template_url!(options)
-    def s3_presigned_url(template_output_path)
-      template_path = template_output_path.sub('output/templates/','')
-      key = "#{@prefix}/#{template_path}"
-      s3_presigner.presigned_url(:get_object, bucket: s3_bucket, key: key)
-    end
-
-    # Parse the s3_folder setting and remove the folder portion to leave the
-    # "s3_bucket" portion
-    # Example:
-    #    s3_bucket('s3://mybucket/templates/storage/path')
-    #    => mybucket
-    def s3_bucket
-      Lono::S3::Bucket.name
-    end
-
-    def say(message)
-      puts message unless @options[:quiet]
+      puts "Templates uploaded to s3."
     end
   end
-end
+end

data/lib/lono/utils/item/file_methods.rb

@@ -0,0 +1,29 @@
+module Lono::Utils::Item
+  module FileMethods
+    def exist?
+      File.exist?(output_path)
+    end
+
+    def directory?
+      File.directory?(output_path)
+    end
+
+    def file?
+      File.file?(output_path)
+    end
+
+    def s3_path
+      path = zip_file_path.gsub("#{Lono.root}/",'') # remove Lono.root
+      "#{Lono.env}/#{path}"
+    end
+
+    # full path
+    def zip_file_path
+      "#{File.dirname(output_path)}/#{zip_file_name}"
+    end
+
+    def zip_file_name
+      "#{File.basename(output_path)}-#{@type}-#{Lono::Md5.sum(output_path)}.zip"
+    end
+  end
+end

data/lib/lono/utils/item/zip.rb

@@ -0,0 +1,42 @@
+require 'fileutils'
+
+module Lono::Utils::Item
+  module Zip
+    def zip(item)
+      if item.directory?
+        zip_directory(item)
+      else
+        zip_file(item)
+      end
+    end
+
+    def zip_file(item)
+      path = item.output_path
+      zip_file = item.zip_file_name
+
+      puts "Zipping file and generating md5 named file from: #{path}"
+      command = "cd #{File.dirname(path)} && zip -q #{zip_file} #{File.basename(path)}" # create zipfile at same level of file
+      execute_zip(command)
+    end
+
+    def zip_directory(item)
+      path = item.output_path
+      zip_file = item.zip_file_name
+
+      puts "Zipping folder and generating md5 named file from: #{path}"
+      command = "cd #{path} && zip --symlinks -rq #{zip_file} ." # create zipfile witih directory
+      execute_zip(command)
+      FileUtils.mv("#{path}/#{zip_file}", "#{File.dirname(path)}/#{zip_file}") # move zip back to the parent directory
+    end
+
+    def execute_zip(command)
+      # puts "=> #{command}".color(:green) # uncomment to debug
+      `#{command}`
+      unless $?.success?
+        puts "ERROR: Fail to run #{command}".color(:red)
+        puts "Maybe zip is not installed or path is incorrect?"
+        exit 1
+      end
+    end
+  end
+end

data/lib/lono/utils/rsync.rb

@@ -0,0 +1,36 @@
+require 'shellwords'
+
+module Lono::Utils
+  module Rsync
+    def sh(command)
+      puts "=> #{command}"
+      out = `#{command}`
+      puts out if ENV['LONO_DEBUG_SH']
+      success = $?.success?
+      raise("ERROR: running command #{command}").color(:red) unless success
+      success
+    end
+
+    def rsync(src, dest)
+      # Using FileUtils.cp_r doesnt work if there are special files like socket files in the src dir.
+      # Instead of using this hack https://bugs.ruby-lang.org/issues/10104
+      # Using rsync to perform the copy.
+      src.chop! if src.ends_with?('/')
+      dest.chop! if dest.ends_with?('/')
+      check_rsync_installed!
+      # Ensures required trailing slashes
+      FileUtils.mkdir_p(File.dirname(dest))
+      sh "rsync -a --links --no-specials --no-devices #{Shellwords.escape(src)}/ #{Shellwords.escape(dest)}/"
+    end
+
+    @@rsync_installed = false
+    def check_rsync_installed!
+      return if @@rsync_installed # only check once
+      if system "type rsync > /dev/null 2>&1"
+        @@rsync_installed = true
+      else
+        raise "ERROR: Rsync is required. Rsync does not seem to be installed.".color(:red)
+      end
+    end
+  end
+end

data/lib/lono/version.rb

@@ -1,3 +1,3 @@
 module Lono
-  VERSION = "7.3.2"
+  VERSION = "7.4.0"
 end

data/lib/templates/blueprint/.gitignore

@@ -1,14 +1,11 @@
 /.bundle/
+/.rspec_status
 /.yardoc
 /_yardoc/
 /coverage/
 /doc/
+/Gemfile.lock
+/output
 /pkg/
 /spec/reports/
-/tmp/
-
-# rspec failure tracking
-.rspec_status
-
-output
-Gemfile.lock
+/tmp/

data/lib/templates/configset/%configset_name%.gemspec.tt

@@ -7,7 +7,7 @@ Gem::Specification.new do |spec|
   spec.summary       = "Write a short summary because it's required."  # TODO: Change me
   spec.description   = "Write a longer description or delete this line."  # TODO: Change me
   spec.homepage      = "<%= ENV['LONO_ORG'] || "https://github.com/USER" %>/<%= configset_name %>"
-  spec.license       = "<%= ENV['LONO_LICENSE'] || '' %>"
+  spec.license       = "<%= ENV['LONO_LICENSE'] || 'Nonstandard' %>"
 
   # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
   # to allow pushing to a single host or delete this section to allow pushing to any host.

data/lib/templates/configset/.gitignore

@@ -1,12 +1,10 @@
 /.bundle/
+/.rspec_status
 /.yardoc
 /_yardoc/
 /coverage/
 /doc/
+/Gemfile.lock
 /pkg/
 /spec/reports/
 /tmp/
-
-# rspec failure tracking
-.rspec_status
-Gemfile.lock

data/lib/templates/extension/%extension_name%.gemspec.tt

@@ -36,7 +36,8 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "activesupport"
+  spec.add_dependency "activesupport", "~> 6"
+  spec.add_dependency "zeitwerk", "~> 2"
 
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "byebug"

data/lib/templates/extension/.gitignore

@@ -1,17 +1,11 @@
-*.gem
-*.rbc
-.bundle
-.config
-.yardoc
-_yardoc
-coverage
-doc/
-InstalledFiles
-lib/bundler/man
-pkg
-rdoc
-spec/reports
-test/tmp
-test/version_tmp
-tmp
-Gemfile.lock
+/.bundle/
+/.rspec_status
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/Gemfile.lock
+/output
+/pkg/
+/spec/reports/
+/tmp/

data/lib/templates/extension/lib/%extension_name%.rb.tt

@@ -1,5 +1,8 @@
 $:.unshift(File.expand_path("../", __FILE__))
 require "<%= extension_underscore_name %>/version"
 
+require "<%= extension_underscore_name %>/autoloader"
+<%= extension_class_name %>::Autoloader.setup
+
 module <%= extension_class_name %>
 end

data/lib/templates/extension/lib/%extension_name%/autoloader.rb.tt

@@ -0,0 +1,23 @@
+require "zeitwerk"
+
+module <%= extension_class_name %>
+  class Autoloader
+    class Inflector < Zeitwerk::Inflector
+      def camelize(basename, _abspath)
+        map = { cli: "CLI", version: "VERSION" }
+        map[basename.to_sym] || super
+      end
+    end
+
+    class << self
+      def setup
+        loader = Zeitwerk::Loader.new
+        loader.inflector = Inflector.new
+        loader.push_dir(File.dirname(__dir__)) # lib
+        # loader.log!
+        loader.ignore("#{__dir__}/user_data")
+        loader.setup
+      end
+    end
+  end
+end

data/lono.gemspec

@@ -27,6 +27,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency "aws-sdk-iam" # lono seed
   gem.add_dependency "aws-sdk-s3"
   gem.add_dependency "aws-sdk-ssm"
+  gem.add_dependency "aws_data"
   gem.add_dependency "bundler", "~> 2"
   gem.add_dependency "cfn_camelizer"
   gem.add_dependency "cli-format"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: lono
 version: !ruby/object:Gem::Version
-  version: 7.3.2
+  version: 7.4.0
 platform: ruby
 authors:
 - Tung Nguyen
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-03-17 00:00:00.000000000 Z
+date: 2020-03-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -108,6 +108,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws_data
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -504,13 +518,19 @@ files:
 - lib/lono/configset/register/dsl.rb
 - lib/lono/configset/register/project.rb
 - lib/lono/configset/resolver.rb
+- lib/lono/configset/s3_file/build.rb
+- lib/lono/configset/s3_file/item.rb
+- lib/lono/configset/s3_file/registry.rb
+- lib/lono/configset/s3_file/upload.rb
 - lib/lono/configset/strategy/base.rb
 - lib/lono/configset/strategy/dsl.rb
-- lib/lono/configset/strategy/dsl/helpers.rb
-- lib/lono/configset/strategy/dsl/helpers/core.rb
-- lib/lono/configset/strategy/dsl/syntax.rb
 - lib/lono/configset/strategy/erb.rb
-- lib/lono/configset/strategy/erb/helpers.rb
+- lib/lono/configset/strategy/helpers/dsl.rb
+- lib/lono/configset/strategy/helpers/dsl/auth.rb
+- lib/lono/configset/strategy/helpers/dsl/core.rb
+- lib/lono/configset/strategy/helpers/dsl/package.rb
+- lib/lono/configset/strategy/helpers/dsl/syntax.rb
+- lib/lono/configset/strategy/helpers/erb.rb
 - lib/lono/conventions.rb
 - lib/lono/core.rb
 - lib/lono/core/config.rb
@@ -609,6 +629,7 @@ files:
 - lib/lono/registration/user.rb
 - lib/lono/s3.rb
 - lib/lono/s3/bucket.rb
+- lib/lono/s3/uploader.rb
 - lib/lono/script.rb
 - lib/lono/script/base.rb
 - lib/lono/script/build.rb
@@ -664,6 +685,7 @@ files:
 - lib/lono/template/strategy/dsl/builder/fn.rb
 - lib/lono/template/strategy/dsl/builder/helpers.rb
 - lib/lono/template/strategy/dsl/builder/helpers/core_helper.rb
+- lib/lono/template/strategy/dsl/builder/helpers/ec2_helper.rb
 - lib/lono/template/strategy/dsl/builder/helpers/file_helper.rb
 - lib/lono/template/strategy/dsl/builder/helpers/lookup_helper.rb
 - lib/lono/template/strategy/dsl/builder/helpers/s3_helper.rb
@@ -691,7 +713,10 @@ files:
 - lib/lono/template/util.rb
 - lib/lono/upgrade.rb
 - lib/lono/user_data.rb
+- lib/lono/utils/item/file_methods.rb
+- lib/lono/utils/item/zip.rb
 - lib/lono/utils/pretty_time.rb
+- lib/lono/utils/rsync.rb
 - lib/lono/utils/sure.rb
 - lib/lono/version.rb
 - lib/lono/yamler/loader.rb
@@ -725,6 +750,7 @@ files:
 - lib/templates/extension/Gemfile.tt
 - lib/templates/extension/Rakefile.tt
 - lib/templates/extension/lib/%extension_name%.rb.tt
+- lib/templates/extension/lib/%extension_name%/autoloader.rb.tt
 - lib/templates/extension/lib/%extension_name%/helpers/mappings.rb.tt
 - lib/templates/extension/lib/%extension_name%/helpers/outputs.rb.tt
 - lib/templates/extension/lib/%extension_name%/helpers/parameters.rb.tt