lono 7.5.2 → 8.0.0.pre.rc3

data/lib/lono/{template/strategy/dsl/builder → builder/util}/stringify.rb

@@ -1,4 +1,4 @@
-class Lono::Template::Strategy::Dsl::Builder
+module Lono::Builder::Util
   module Stringify
     # Accounts for Arrays also. ActiveSupport only works for Hashes.
     def stringify!(data)

data/lib/lono/bundler/cli/base.rb

@@ -0,0 +1,10 @@
+class Lono::Bundler::CLI
+  class Base
+    include LB::Util::Logging
+    include LB::Util::Sure
+
+    def initialize(options={})
+      @options = options
+    end
+  end
+end

data/lib/lono/bundler/cli/bundle.rb

@@ -0,0 +1,44 @@
+class Lono::Bundler::CLI
+  class Bundle < Lono::Command
+    lonofile_option = Proc.new {
+      option :lonofile, default: ENV['LONOFILE'] || "Lonofile", desc: "Lonofile to use"
+    }
+
+    desc "list", "List bundled blueprints included by Lonofile"
+    long_desc Help.text("bundle/list")
+    lonofile_option.call
+    def list
+      Lono::Bundler::List.new(options).run
+    end
+
+    desc "info COMPONENT", "Provide info about a bundled component"
+    long_desc Help.text("bundle/info")
+    lonofile_option.call
+    option :type, aliases: :t, desc: "Type. IE: blueprint, configset, extension"
+    def info(component)
+      Lono::Bundler::Info.new(options.merge(component: component)).run
+    end
+
+    desc "install", "Install blueprints from the Lonofile"
+    long_desc Help.text("bundle/install")
+    lonofile_option.call
+    def install
+      Lono::Bundler::Runner.new(options).run
+    end
+
+    desc "clean", "Clean cache"
+    long_desc Help.text("bundle/clean")
+    option :yes, aliases: :y, type: :boolean, desc: "bypass are you sure prompt"
+    def clean
+      Clean.new(options).run
+    end
+
+    desc "update [COMPONENT]", "Update bundled blueprints"
+    long_desc Help.text("bundle/update")
+    lonofile_option.call
+    def update(*components)
+      Lono::Bundler.update_mode = true
+      Lono::Bundler::Runner.new(options.merge(components: components)).run
+    end
+  end
+end

data/lib/lono/bundler/cli/clean.rb

@@ -0,0 +1,27 @@
+class Lono::Bundler::CLI
+  class Clean < Base
+    include LB::Component::Concerns::PathConcern
+
+    def run
+      paths = [tmp_root]
+      are_you_sure?(paths)
+      paths.each do |path|
+        FileUtils.rm_rf(path)
+        logger.info "Removed #{path}"
+      end
+
+    end
+
+    def are_you_sure?(paths)
+      pretty_paths = paths.map { |p| "    #{p}" }.join("\n")
+      message = <<~EOL.chomp
+        Will remove these folders and all their files:
+
+        #{pretty_paths}
+
+        Are you sure?
+      EOL
+      sure?(message) # from Util::Sure
+    end
+  end
+end

data/lib/lono/bundler/cli/help/bundle.md

@@ -0,0 +1,3 @@
+## Examples
+
+    lono bundle

data/lib/lono/bundler/cli/help.rb

@@ -0,0 +1,11 @@
+class Lono::Bundler::CLI
+  module Help
+    class << self
+      def text(namespaced_command)
+        path = namespaced_command.to_s.gsub(':','/')
+        path = File.expand_path("../help/#{path}.md", __FILE__)
+        IO.read(path) if File.exist?(path)
+      end
+    end
+  end
+end

data/lib/lono/bundler/cli.rb

@@ -0,0 +1,7 @@
+module Lono::Bundler
+  class CLI < Lono::Command
+    desc "bundle SUBCOMMAND", "bundle subcommands"
+    long_desc Help.text(:bundle)
+    subcommand "bundle", Bundle
+  end
+end

data/lib/lono/bundler/component/concerns/local_concern.rb

@@ -0,0 +1,10 @@
+module Lono::Bundler::Component::Concerns
+  module LocalConcern
+    def local?
+      source.starts_with?("/")  ||
+      source.starts_with?(".")  ||
+      source.starts_with?("..") ||
+      source.starts_with?("~")
+    end
+  end
+end

data/lib/lono/bundler/component/concerns/notation_concern.rb

@@ -0,0 +1,59 @@
+require 'uri'
+
+module Lono::Bundler::Component::Concerns
+  module NotationConcern
+    def remove_notations(source)
+      remove_subfolder_notation(remove_ref_notation(source))
+    end
+
+    def remove_ref_notation(source)
+      source.sub(/\?.*/,'')
+    end
+
+    def remove_subfolder_notation(source)
+      clean = clean_notation(source)
+      parts = clean.split('//')
+      if parts.size == 2 # has subfolder
+        source.split('//')[0..-2].join('//') # remove only subfolder, keep rest of original source
+      else
+        source
+      end
+    end
+
+    def subfolder(source)
+      clean = clean_notation(source)
+      parts = clean.split('//')
+      if parts.size == 2 # has subfolder
+        remove_ref_notation(parts.last)
+      end
+    end
+
+    def ref(source)
+      url = clean_notation(source)
+      uri = URI(url)
+      if uri.query
+        params = URI::decode_www_form(uri.query).to_h # if you are in 2.1 or later version of Ruby
+        params['ref']
+      end
+    end
+
+
+    def clean_notation(source)
+      clean = source
+        .sub(/.*::/,'')            # remove git::
+        .sub(%r{http[s?]://},'')   # remove https://
+        .sub(%r{git@(.*?):},'')    # remove git@word
+      remove_host(clean)
+    end
+
+    def remove_host(clean)
+      return clean unless clean.include?('ssh://')
+      if clean.count(':') == 1
+        uri = URI(clean)
+        uri.path
+      else
+        clean.split(':').last
+      end
+    end
+  end
+end

data/lib/lono/bundler/component/concerns/path_concern.rb

@@ -0,0 +1,68 @@
+module Lono::Bundler::Component::Concerns
+  module PathConcern
+    def setup_tmp
+      FileUtils.mkdir_p(cache_root)
+      FileUtils.mkdir_p(stage_root)
+    end
+
+    def tmp_root
+      "/tmp/lono/bundler"
+    end
+
+    def cache_root
+      "#{tmp_root}/cache"
+    end
+
+    def stage_root
+      "#{tmp_root}/stage"
+    end
+
+    def cache_path(name)
+      [cache_root, @component.source_type, name].compact.join('/')
+    end
+
+    def stage_path(name)
+      [stage_root, @component.source_type, name].compact.join('/')
+    end
+
+    # Fetcher: Downloader/Local copies to a slightly different folder.
+    # Also, Copy will use this and reference same method so it's consistent.
+    def rel_dest_dir
+      case @component.source_type
+      when 'local'
+        @component.name      # example-module
+      when 's3'
+        path = type_path # https://s3-us-west-2.amazonaws.com/demo-terraform-test/example-module.zip
+        remove_ext(path) # demo-terraform-test/blueprints/example-module
+      when 'gcs'
+        path = type_path # https://www.googleapis.com/storage/v1/BUCKET_NAME/PATH/TO/module.zip
+        path.sub!(%r{storage/v\d+/},'')
+        remove_ext(path) # terraform-example-blueprints/blueprints/example-module
+      when 'http'
+        path = type_path # https://www.googleapis.com/storage/v1/BUCKET_NAME/PATH/TO/module.zip
+        remove_ext(path) # terraform-example-blueprints/blueprints/example-module
+      else # inferred git, registry, git::, ssh://, git::ssh://
+        @component.repo_folder #  tongueroo/example-module
+      end
+    end
+
+    def type_path
+      source = @component.source
+      url = source.sub("#{@component.source_type}::",'')
+      uri = URI(url)
+      uri.path.sub('/','')   # removing leading slash to bucket name is the first thing
+         .sub(%r{//(.*)},'') # remove subfolder
+    end
+
+    def remove_ext(path)
+      ext = File.extname(path)
+      path.sub(ext,'')
+    end
+
+    def get_bucket_key(path)
+      bucket, *rest = path.split('/')
+      key = rest.join('/')
+      [bucket, key]
+    end
+  end
+end

data/lib/lono/bundler/component/concerns/stack_concern.rb

@@ -0,0 +1,60 @@
+module Lono::Bundler::Component::Concerns
+  module StackConcern
+    def stacks
+      stacks = @props[:stacks] || @props[:stack]
+      return unless stacks
+      if all_stacks?(stacks)
+        stacks = all_stacks
+      elsif stacks.is_a?(Array)
+        stacks = array_stacks(stacks)
+      end
+      normalize_stacks(stacks)
+    end
+    alias_method :stack, :stacks # important to have alias for VersionCheck
+
+    def all_stacks?(*stacks)
+      stacks.flatten == [:all]
+    end
+
+    def all_stacks
+      stack = Lono::Bundler::Exporter::Stacks::Stack.new(self) # to get the component src path
+      expr = "#{stack.examples_folder}/*"
+      dirs = Dir.glob(expr).select { |path| File.directory?(path) }
+      dirs.map do |dir|
+        example = File.basename(dir)
+        # Set name so multiple app/stacks are created instead of just one app/stack/component
+        {
+          name: example,
+          src: example,
+        }
+      end
+    end
+
+    def array_stacks(stacks)
+      stacks.map do |example|
+        if example.is_a?(Hash)
+          example
+        else
+          {
+            name: example,
+            src: example,
+          }
+        end
+      end
+    end
+
+    # Normalizes stack options to an Array of Hashes or just a single Hash
+    def normalize_stacks(option)
+      defaults = LB.config.stack_options.dup
+      result = case option
+      when String
+        [defaults.merge(src: option)]
+      when Array
+        option.map! {|s| normalize_stacks(s) }
+      else # Hash
+        [defaults.merge!(option)]
+      end
+      result.flatten
+    end
+  end
+end

data/lib/lono/bundler/component/fetcher/base.rb

@@ -0,0 +1,27 @@
+# Interface of subclasses should implement
+#
+#   run
+#   switch_version(component.sha)
+#   sha
+#
+class Lono::Bundler::Component::Fetcher
+  class Base
+    include LB::Util::Git
+    include LB::Util::Logging
+    include LB::Component::Concerns::PathConcern
+
+    attr_reader :sha # returns nil for Local
+    def initialize(component)
+      @component = component
+    end
+
+    def switch_version(*)
+      # noop
+    end
+
+    def extract(archive, dest)
+      Lono::Bundler::Extract.extract(archive, dest)
+    end
+
+  end
+end

data/lib/lono/bundler/component/fetcher/gcs.rb

@@ -0,0 +1,69 @@
+begin
+  require "google/cloud/storage"
+rescue LoadError => e
+  $stderr.puts "#{e.class}: #{e.message}".color(:red)
+  $stderr.puts <<~EOL
+
+    Unable to: require "google/cloud/storage"
+
+    In order to use gcs as a lono bundler source,
+    please add the google-cloud-storage gem to
+    your Lono project's Gemfile and run bundle.
+
+    To add the gem to your Gemfile, you can run:
+
+        bundle add google-cloud-storage
+
+    Then download blueprints in defined your Lonofile with:
+
+        lono bundle
+  EOL
+  exit 1
+end
+
+class Lono::Bundler::Component::Fetcher
+  class Gcs < Base
+    extend Memoist
+
+    def run
+      bucket, key, path = gcs_info
+      download(bucket, key, path)
+    end
+
+    def download(bucket_name, key, path)
+      # Download to cache area
+      bucket = storage.bucket(bucket_name)
+      unless bucket
+        logger.error "ERROR: bucket #{bucket_name} does not exist".color(:red)
+        exit 1
+      end
+      file = bucket.file(key)
+      unless file
+        logger.error "ERROR: Unable to find gs://#{bucket_name}/#{key}".color(:red)
+        exit 1
+      end
+
+      archive = cache_path(path) # temporary path
+      logger.debug "Gcs save archive to #{archive}"
+      FileUtils.mkdir_p(File.dirname(archive))
+      file.download(archive)
+
+      # Save to stage
+      dest = stage_path(rel_dest_dir)
+      extract(archive, dest)
+    end
+
+  private
+    def gcs_info
+      path = type_path
+      path.sub!(%r{storage/v\d+/},'')
+      bucket, key = get_bucket_key(path)
+      [bucket, key, path]
+    end
+
+    def storage
+      Google::Cloud::Storage.new
+    end
+    memoize :storage
+  end
+end

data/lib/lono/bundler/component/fetcher/git.rb

@@ -0,0 +1,69 @@
+class Lono::Bundler::Component::Fetcher
+  class Git < Base
+    extend Memoist
+
+    def run
+      setup_tmp
+      org_folder = cache_path(@component.org_folder)
+      FileUtils.mkdir_p(org_folder)
+
+      Dir.chdir(org_folder) do
+        logger.debug "Current root dir: #{org_folder}"
+        clone unless File.exist?(@component.repo)
+
+        Dir.chdir(@component.repo) do
+          logger.debug "Change dir: #{@component.repo}"
+          git "pull"
+          git "submodule update --init"
+          stage
+        end
+      end
+    end
+    memoize :run
+
+    def clone
+      command = ["git clone", ENV['LB_GIT_CLONE_ARGS'], @component.url].compact.join(' ')
+      sh command
+    rescue LB::GitError => e
+      logger.error "ERROR: #{e.message}".color(:red)
+      exit 1
+    end
+
+    def stage
+      copy_to_stage
+      checkout = @component.checkout_version || default_branch
+      switch_version(checkout)
+    end
+
+    # Note: if not in a git repo, will get this error message in stderr
+    #
+    #    fatal: Not a git repository (or any of the parent directories): .git
+    #
+    def default_branch
+      origin = `git remote show origin`.strip
+      found = origin.split("\n").find { |l| l.include?("HEAD") }
+      if found
+        found.split(':').last.strip
+      else
+        ENV['TS_GIT_DEFAULT_BRANCH'] || 'master'
+      end
+    end
+
+    def switch_version(version)
+      stage_path = stage_path(rel_dest_dir)
+      logger.debug "stage_path #{stage_path}"
+      Dir.chdir(stage_path) do
+        git "checkout #{version}"
+        @sha = git("rev-parse HEAD").strip
+      end
+    end
+
+    def copy_to_stage
+      cache_path = cache_path(rel_dest_dir)
+      stage_path = stage_path(rel_dest_dir)
+      FileUtils.rm_rf(stage_path)
+      FileUtils.mkdir_p(File.dirname(stage_path))
+      FileUtils.cp_r(cache_path, stage_path)
+    end
+  end
+end

data/lib/lono/bundler/component/fetcher/local.rb

@@ -0,0 +1,15 @@
+class Lono::Bundler::Component::Fetcher
+  class Local < Base
+    def run
+      stage_path = stage_path(rel_dest_dir)
+      source = @component.source
+      src = source.sub(/^~/, ENV['HOME']) # allow ~/ notation
+      FileUtils.rm_rf(stage_path)
+      FileUtils.mkdir_p(File.dirname(stage_path))
+      logger.debug "Local: cp -r #{src} #{stage_path}"
+      # copy from stage area to vendor/blueprints/NAME
+      # IE: cp -r /tmp/lono/bundler/stage/local/local1 vendor/blueprints/local1
+      FileUtils.cp_r(src, stage_path)
+    end
+  end
+end

data/lib/lono/bundler/component/fetcher/s3.rb

@@ -0,0 +1,66 @@
+require 'aws-sdk-s3'
+require 'uri'
+
+class Lono::Bundler::Component::Fetcher
+  class S3 < Base
+    extend Memoist
+
+    def run
+      region, bucket, key, path = s3_info
+      download(region, bucket, key, path)
+    end
+
+    def download(region, bucket, key, path)
+      # Download to cache area
+      response_target = cache_path(path) # temporary path
+
+      unless File.exist?(response_target)
+        logger.debug "S3 save archive to #{response_target}".color(:yellow)
+        FileUtils.mkdir_p(File.dirname(response_target))
+        s3_get(region, response_target, bucket, key)
+      end
+
+      # Save to stage
+      dest = stage_path(rel_dest_dir)
+      extract(response_target, dest)
+    end
+
+  private
+    def s3_get(region, response_target, bucket, key)
+      s3(region).get_object(
+        response_target: response_target,
+        bucket: bucket,
+        key: key,
+      )
+    rescue Aws::S3::Errors::NoSuchBucket, Aws::S3::Errors::NoSuchKey => e
+      logger.error(<<~EOL.color(:red))
+        ERROR: #{e.class} #{e.message}
+
+            bucket: #{bucket}
+            key: #{key}
+
+      EOL
+      exit 1
+    end
+
+    def s3_info
+      path = type_path
+      bucket, key = get_bucket_key(path)
+
+      url = @component.source.sub('s3::','')
+      uri = URI(url)
+      region = if uri.host == 'https://s3.amazonaws.com'
+                 'us-east-1'
+               else
+                 uri.host.match(/s3-(.*)\.amazonaws/)[1]
+               end
+
+      [region, bucket, key, path]
+    end
+
+    def s3(region)
+      Aws::S3::Client.new(region: region)
+    end
+    memoize :s3
+  end
+end

data/lib/lono/bundler/component/fetcher.rb

@@ -0,0 +1,18 @@
+# Delegates to:
+#
+#   1. Local
+#   2. Git
+#
+class Lono::Bundler::Component
+  class Fetcher
+    def initialize(component)
+      @component = component
+    end
+
+    def instance
+      type = @component.source_type == "registry" ? "git" : @component.source_type
+      klass = "Lono::Bundler::Component::Fetcher::#{type.camelize}".constantize
+      klass.new(@component) # IE: Local.new(@component), Git.new(@component), S3.new(@component), etc
+    end
+  end
+end

data/lib/lono/bundler/component/http/concern.rb

@@ -0,0 +1,45 @@
+require 'net/http'
+require 'open-uri'
+
+module Lono::Bundler::Component::Http
+  module Concern
+    include LB::Util::Logging
+
+    def http_request(url, auth_domain: nil)
+      uri = URI(url)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = uri.scheme == "https"
+      # Total time will be 40s = 20 x 2
+      http.max_retries = 1 # Default is already 1, just  being explicit
+      http.read_timeout = 20 # Sites that dont return in 20 seconds are considered down
+      request = Net::HTTP::Get.new(uri)
+
+      if auth_domain
+        path = "#{ENV['HOME']}/.terraform.d/credentials.tfrc.json"
+        if File.exist?(path)
+          data = JSON.load(IO.read(path))
+          token = data['credentials'][auth_domain]['token']
+          request.add_field 'Authorization', "Bearer #{token}"
+        else
+          auth_error_exit!
+        end
+      end
+
+      begin
+         http.request(request) # response
+      rescue Net::OpenTimeout => e # internal ELB but VPC is not configured for Lambda function
+        http_request_error_message(e)
+      rescue Exception => e
+        # Net::ReadTimeout - too slow
+        # Errno::ECONNREFUSED - completely down
+        # SocketError - improper url "dsfjsl" instead of example.com
+        http_request_error_message(e)
+      end
+    end
+
+    def http_request_error_message(e)
+      logger.info "ERROR: #{e.message}\n#{e.message}".color(:red)
+      exit 1
+    end
+  end
+end

data/lib/lono/bundler/component/http/source.rb

@@ -0,0 +1,19 @@
+require 'net/http'
+require 'open-uri'
+
+module Lono::Bundler::Component::Http
+  class Source
+    include LB::Util::Logging
+    include Concern
+
+    def initialize(params)
+      @params = params
+      @options = params[:options]
+      @source = @options[:source]
+    end
+
+    def url
+      @source
+    end
+  end
+end