lono 7.4.11 → 8.0.0.pre.rc1

data/lib/lono/bundler/component/http/concern.rb

@@ -0,0 +1,45 @@
+require 'net/http'
+require 'open-uri'
+
+module Lono::Bundler::Component::Http
+  module Concern
+    include LB::Util::Logging
+
+    def http_request(url, auth_domain: nil)
+      uri = URI(url)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = uri.scheme == "https"
+      # Total time will be 40s = 20 x 2
+      http.max_retries = 1 # Default is already 1, just  being explicit
+      http.read_timeout = 20 # Sites that dont return in 20 seconds are considered down
+      request = Net::HTTP::Get.new(uri)
+
+      if auth_domain
+        path = "#{ENV['HOME']}/.terraform.d/credentials.tfrc.json"
+        if File.exist?(path)
+          data = JSON.load(IO.read(path))
+          token = data['credentials'][auth_domain]['token']
+          request.add_field 'Authorization', "Bearer #{token}"
+        else
+          auth_error_exit!
+        end
+      end
+
+      begin
+         http.request(request) # response
+      rescue Net::OpenTimeout => e # internal ELB but VPC is not configured for Lambda function
+        http_request_error_message(e)
+      rescue Exception => e
+        # Net::ReadTimeout - too slow
+        # Errno::ECONNREFUSED - completely down
+        # SocketError - improper url "dsfjsl" instead of example.com
+        http_request_error_message(e)
+      end
+    end
+
+    def http_request_error_message(e)
+      logger.info "ERROR: #{e.message}\n#{e.message}".color(:red)
+      exit 1
+    end
+  end
+end

data/lib/lono/bundler/component/http/source.rb

@@ -0,0 +1,19 @@
+require 'net/http'
+require 'open-uri'
+
+module Lono::Bundler::Component::Http
+  class Source
+    include LB::Util::Logging
+    include Concern
+
+    def initialize(params)
+      @params = params
+      @options = params[:options]
+      @source = @options[:source]
+    end
+
+    def url
+      @source
+    end
+  end
+end

data/lib/lono/bundler/component/org_repo.rb

@@ -0,0 +1,65 @@
+class Lono::Bundler::Component
+  class OrgRepo
+    def initialize(url)
+      @url = url.sub('ssh://','') # important to copy so dont change the string reference
+    end
+
+    def repo
+      org_repo_words[-1]
+    end
+
+    def org
+      s = org_folder.split('/').last
+      s ? s.split('/').last : 'none'
+    end
+
+    def org_folder
+      org_repo_words[-2] # second to last word
+    end
+
+    def repo_folder
+      org_repo_words.join('/')
+    end
+
+    def org_repo_words
+      if @url.include?(':') && !@url.match(%r{http[s?]://}) # user@host:repo git@github.com:org/repo
+        folder, repo = handle_string_with_colon
+      else # IE: https://github.com/org/repo, org/repo, etc
+        parts = @url.split('/')
+        folder = parts[0..-2].join('/')
+        repo = parts.last
+      end
+
+      org_path = clean_folder(folder)
+      repo = strip_dot_git(repo)
+      [org_path, repo]
+    end
+
+    def clean_folder(folder)
+      folder.sub(%r{.*@},'')           # remove user@
+            .sub(%r{http[s?]://},'')   # remove https://
+    end
+
+    # user@host:repo git@github.com:org/repo
+    def handle_string_with_colon
+      host, path = @url.split(':')
+      if path.size == 2
+        folder, repo = path.split(':')
+      else
+        folder = join(host, File.dirname(path))
+        repo = File.basename(path)
+      end
+      [folder, repo]
+    end
+
+    def join(*path)
+      path.compact!
+      path[1] = path[1].sub('/','') if path[1].starts_with?('/')
+      path.reject(&:blank?).join('/')
+    end
+
+    def strip_dot_git(s)
+      s.sub(/\.git$/,'')
+    end
+  end
+end

data/lib/lono/bundler/component/props/extension.rb

@@ -0,0 +1,19 @@
+class Lono::Bundler::Component::Props
+  module Extension
+    def props(*names)
+      names.each { |n| prop(n) }
+    end
+
+    def prop(name)
+      name = name.to_sym
+      define_method(name) do
+        @props[name]
+      end
+
+      define_method("#{name}=") do |v|
+        @props[name] = v
+      end
+    end
+  end
+end
+

data/lib/lono/bundler/component/props/typer.rb

@@ -0,0 +1,41 @@
+class Lono::Bundler::Component::Props
+  class Typer
+    include Lono::Bundler::Component::Concerns::LocalConcern
+    include Lono::Bundler::Component::Concerns::NotationConcern
+
+    delegate :source, to: :props
+
+    attr_reader :props
+    def initialize(props)
+      @props = props # Props.new object
+    end
+
+    # IE: git or registry
+    def source_type
+      if source.include?('ssh://')
+        "git"
+      elsif source.include?('::')
+        source.split('::').first # IE: git:: s3:: gcs::
+      elsif local?
+        "local"
+      elsif registry?
+        "registry"
+      else
+        "git"
+      end
+    end
+
+  private
+    # dont use registry? externally. instead use type since it can miss local detection
+    def registry?
+      if source.nil? ||
+         source.starts_with?('git@') || # git@github.com:tongueroo/pet
+         source.starts_with?('http') || # https://github.com/tongueroo/pet
+         source.include?('::')          # git::https:://git.example.com/pet
+         return false
+      end
+      s = remove_notations(@props.source)
+      s.split('/').size == 3 || s.split('/').size == 4
+    end
+  end
+end

data/lib/lono/bundler/component/props.rb

@@ -0,0 +1,122 @@
+# Use to build component from Lonofile entry.
+# The Lonofile.lock does not need this because it's simple and just merges the data.
+class Lono::Bundler::Component
+  class Props
+    extend Memoist
+    include Concerns::NotationConcern
+    include LB::Util::Logging
+
+    delegate :source_type, to: :typer
+
+    attr_reader :source
+    def initialize(options={})
+      @options = options
+      @source, @version = @options[:source], @options[:version]
+    end
+
+    def name
+      remove_notations(@options[:name])
+    end
+
+    def build
+      unless @source
+        logger.error "ERROR: The source option must be set for the #{name} component in the Lonofile".color(:red)
+        exit 1
+      end
+      o = @options.merge(
+        name: name,
+        source_type: source_type,
+        url: url,
+      )
+      o[:subfolder] ||= subfolder(@source)
+      o[:ref] ||= ref(@source)
+      o.deep_symbolize_keys
+    end
+
+    # url is normalized
+    def url
+      url = case source_type
+            when 'registry'
+              registry.github_url
+            when 'local'
+              source
+            when 'http'
+              http_source_url
+            else # git
+              git_source_url
+            end
+      remove_notations(clone_with(url))
+    end
+
+    # apply clone_with option if set
+    def clone_with(url)
+      with = @options[:clone_with] || LB.config.clone_with
+      return url unless with
+      if with == 'https'
+        url.sub(/.*@(.*):/,'https://\1/')
+      else # git@
+        url.sub(%r{http[s]?://(.*?)/},'git@\1:')
+      end
+    end
+
+    def http_source_url
+      source = Http::Source.new(@params)
+      source.url
+    end
+
+    # git_source_url is normalized
+    #
+    # See: https://stackoverflow.com/questions/6167905/git-clone-through-ssh
+    #
+    #    ssh://username@host.xz/absolute/path/to/repo.git/ - just a forward slash for absolute path on server
+    #    username@host.xz:relative/path/to/repo.git/ - just a colon (it mustn't have the ssh:// for relative path on server (relative to home dir of username on server machine)
+    #
+    # When colon is separator for relative path do not want ssh prepended
+    #
+    #   git clone ec2-user@localhost:repo
+    #
+    # When slash is separator for absolute path want ssh prepended
+    #
+    #   git clone ssh://ec2-user@localhost/path/to/repo => valid URI
+    #
+    def git_source_url
+      if @source.include?('http') || @source.include?(':')
+        # Examples:
+        #   component "pet1", source: "https://github.com/tongueroo/pet"
+        #   component "pet2", source: "git@github.com:tongueroo/pet"
+        #   component "pet3", source: "git@gitlab.com:tongueroo/pet"
+        #   component "pet4", source: "git@bitbucket.org:tongueroo/pet"
+        #   component "example3", source: "git::https://example.com/example-module.git"
+        #
+        # sub to allow for generic git repo notiation
+        source = @source.sub('git::','')
+        if source.include?('ssh://')
+          if source.count(':') == 1
+            return source
+          else
+            return source.sub('ssh://', '')
+          end
+        end
+        source
+      else
+        # Examples:
+        #   component "pet", source: "tongueroo/pet"
+        # Or:
+        #   org "tongueroo"
+        #   component "pet", source: "pet"
+        org_source = @source.include?('/') ? @source : "#{LB.config.org}/#{@source}" # adds inferred org
+        "#{LB.config.base_clone_url}#{org_source}"
+      end
+    end
+
+    def typer
+      Typer.new(self)
+    end
+    memoize :typer
+
+    def registry
+      Registry.new(@source, @version)
+    end
+    memoize :registry
+  end
+end

data/lib/lono/bundler/component/registry.rb

@@ -0,0 +1,136 @@
+class Lono::Bundler::Component
+  class Registry
+    include LB::Util::Logging
+    include Http::Concern
+
+    def initialize(source, version)
+      @source, @version = source, version
+    end
+
+    def github_url
+      if @source.split('/').size > 3 # IE: app.terraform.io/demo-qa/s3-webapp/aws
+        private_github_url
+      else # assume size is 3. IE: terraform-aws-blueprints/security-group/aws
+        public_github_url
+      end
+    end
+\
+    # https://www.terraform.io/docs/cloud/api/blueprints.html#get-a-module
+    # GET /organizations/:organization_name/registry-blueprints/:registry_name/:namespace/:name/:provider
+    # GET /organizations/demo-qa/registry-blueprints/private/demo-qa/s3-webapp/aws
+    #
+    #   :organization_name	The name of the organization the module belongs to.
+    #   :namespace	        The namespace of the module. For private blueprints this is the name of the organization that owns the module.
+    #   :name	              The module name.
+    #   :provider	          The module provider.
+    #   :registry-name	    Either public or private.
+    #
+    # Example: app.terraform.io/demo-qa/s3-webapp/aws
+    #
+    #   domain: app.terraform.io (private registry indicator)
+    #   org: demo-qa
+    #   module: s3-webapp
+    #   provider: aws
+    #
+    def private_github_url
+      domain, org, name, provider = @source.split('/')
+      registry_name = 'private'
+      namespace = org
+
+      base_site = "https://#{domain}" # IE: domain = 'app.terraform.io'
+      base_url = "#{base_site}/api/v2"
+      api_url = "#{base_url}/organizations/#{org}/registry-blueprints/#{registry_name}/#{namespace}/#{name}/#{provider}"
+
+      resp = http_request(api_url, auth_domain: domain)
+
+      repo_url = case resp.code.to_i
+      when 200
+        result = JSON.load(resp.body)
+        result['data']['attributes']['vcs-repo']['repository-http-url'] # repo_url
+      when 401
+        auth_error_exit!(resp)
+      else
+        logger.error "ERROR: Unable to lookup up module in Terraform Registry: #{@source}".color(:red)
+        puts "resp.code #{resp.code}"
+        pp resp
+        exit 1
+      end
+
+      clone_with = 'git' # TODO: make configurable
+      unless clone_with == 'https'
+        repo_url.sub!('https://github.com/', 'git@github.com:')
+      end
+
+      repo_url
+    end
+
+    # Lonofile example
+    #
+    #      component "sg", source: "terraform-aws-blueprints/security-group/aws", version: "3.10.0"
+    #
+    # Resources: https://www.terraform.io/docs/registry/api.html
+    #
+    # Latest version:
+    #
+    #     https://registry.terraform.io/v1/blueprints/terraform-aws-blueprints/sqs/aws/2.1.0/download
+    #
+    # The latest version returns an 302 and contains a location header that is followed
+    # and then downloaded the same way the specific version is downloaded.
+    #
+    # Specific version:
+    #
+    #     https://registry.terraform.io/v1/blueprints/terraform-aws-blueprints/sqs/aws/download
+    #
+    # The specific version returns an 204 and then we grab the download url info form the x-terraform-get header.
+    def public_github_url
+      base_site = "https://registry.terraform.io"
+      base_url = "#{base_site}/v1/blueprints"
+      version = get_version(base_url)
+      api_url = [base_url, @source, version, "download"].compact.join('/')
+
+      resp = http_request(api_url)
+
+      case resp.code.to_i
+      when 204 # IE: curl -v https://registry.terraform.io/v1/blueprints/terraform-aws-blueprints/security-group/aws/3.10.0/download
+        download_url = resp.header["x-terraform-get"]
+      else
+        logger.error "ERROR: Unable to lookup up module in Terraform Registry: #{@source}".color(:red)
+        logger.error "api_url: #{api_url}"
+        puts "resp.code #{resp.code}"
+        pp resp
+        exit 1
+      end
+
+      url = download_url.sub(%r{/archive/.*},'')
+      # IE: git::https://github.com/terraform-aws-blueprints/terraform-aws-security-group?ref=v3.10.0
+      url.sub(/^git::/,'').sub(/\?.*/,'')
+    end
+
+    def get_version(base_url)
+      return @version.sub(/^v/,'') if @version # v1.0 => 1.0
+      api_url = [base_url, @source].join('/')
+      resp = http_request(api_url)
+      data = JSON.load(resp.body)
+      data['version']
+    end
+
+  private
+    def auth_error_exit!(resp=nil, domain=nil)
+      logger.error <<~EOL.color(:red)
+        ERROR: Unauthorized. Unable to lookup up module in Terraform Registry:
+
+            #{@source}
+
+        Try logging in with:
+
+            terraform login #{domain}
+
+      EOL
+      if resp
+        puts "resp.code #{resp.code}"
+        pp resp
+      end
+      exit 1
+    end
+  end
+end

data/lib/lono/bundler/component.rb

@@ -0,0 +1,52 @@
+module Lono::Bundler
+  class Component
+    extend Memoist
+    extend Props::Extension
+    props :export_to, :name, :sha, :source, :subfolder, :source_type, :type, :url, :clone_with
+    delegate :repo, :org, :repo_folder, :org_folder, to: :org_repo
+
+    include Concerns::StackConcern
+    include Concerns::LocalConcern
+
+    attr_reader :props, :version, :ref, :tag, :branch
+    def initialize(options={})
+      @props = Component::Props.new(options.deep_symbolize_keys).build
+      # These props are used for version comparing by VersionComparer
+      @version, @ref, @tag, @branch = @props[:version], @props[:ref], @props[:tag], @props[:branch]
+    end
+
+    # support variety of options, prefer version
+    def checkout_version
+      @version || @ref || @tag || @branch
+    end
+
+    def latest_sha
+      fetcher = Fetcher.new(self).instance
+      fetcher.run
+      fetcher.sha
+    end
+
+    def vcs_provider
+      if url.include?('http')
+        # "https://github.com/org/repo"  => github.com
+        url.match(%r{http[s]?://(.*?)/})[1]
+      elsif url.include?('http') # git@
+        # "git@github.com:org/repo"      => github.com
+        url.match(%r{git@(.*?):})[1]
+      else # ssh://user@domain.com/path/to/repo
+        'none'
+      end
+    end
+
+    def export_path
+      export_to = self.export_to || LB.config.export_to
+      "#{export_to}/#{type.pluralize}/#{name}"
+    end
+
+  private
+    def org_repo
+      OrgRepo.new(url)
+    end
+    memoize :org_repo
+  end
+end

data/lib/lono/bundler/config.rb

@@ -0,0 +1,18 @@
+module Lono::Bundler
+  class Config
+    extend Memoist
+    include Singleton
+
+    def config
+      config = ActiveSupport::OrderedOptions.new
+      config.base_clone_url = "https://github.com/"
+      config.export_to = ENV['LB_EXPORT_TO'] || "vendor"
+      config.export_purge = ENV['LB_EXPORT_PRUNE'] == '0' ? false : true
+      config.lonofile = ENV['LB_LONOFILE'] || "Lonofile"
+      config.lockfile = "#{config.lonofile}.lock"
+      config.logger = Lono.config.logger
+      config
+    end
+    memoize :config
+  end
+end

data/lib/lono/bundler/dsl/syntax.rb

@@ -0,0 +1,46 @@
+class Lono::Bundler::Dsl
+  module Syntax
+    def org(value)
+      config.org = value
+    end
+    alias_method :user, :org
+
+    def base_clone_url(value)
+      config.base_clone_url = value
+    end
+
+    def export_to(path)
+      config.export_to = path
+    end
+
+    def export_purge(value)
+      config.export_purge = value
+    end
+
+    def stack_options(value={})
+      config.stack_options.merge!(value)
+    end
+
+    def clone_with(value)
+      config.clone_with = value
+    end
+
+    def config
+      LB.config
+    end
+
+    def component(name, options={})
+      meta[:components] << options.merge(name: name)
+    end
+
+    def self.friendly_method(meth)
+      define_method meth do |name, options={}|
+        component(name, options.merge(type: meth.to_s))
+      end
+    end
+
+    friendly_method :blueprint
+    friendly_method :configset
+    friendly_method :extension
+  end
+end

data/lib/lono/bundler/dsl.rb

@@ -0,0 +1,21 @@
+module Lono::Bundler
+  class Dsl
+    include DslEvaluator
+    include Syntax
+
+    class_attribute :meta, default: {global: {}, components: []}
+
+    def run
+      evaluate_file(LB.config.lonofile)
+      self
+    end
+
+    def meta
+      self.class.meta
+    end
+
+    def global
+      meta[:global]
+    end
+  end
+end

data/lib/lono/bundler/exporter/base.rb

@@ -0,0 +1,6 @@
+class Lono::Bundler::Exporter
+  class Base
+    include LB::Component::Concerns::PathConcern
+    include LB::Util::Logging
+  end
+end

data/lib/lono/bundler/exporter/copy.rb

@@ -0,0 +1,22 @@
+class Lono::Bundler::Exporter
+  class Copy < Base
+    def initialize(component)
+      @component = component
+    end
+
+    def component
+      FileUtils.rm_rf(@component.export_path)
+      FileUtils.mkdir_p(File.dirname(@component.export_path))
+      logger.debug "Copy: cp -r #{src_path} #{@component.export_path}"
+      FileUtils.cp_r(src_path, @component.export_path)
+      FileUtils.rm_rf("#{@component.export_path}/.git")
+    end
+
+    # src path is from the stage area
+    def src_path
+      path = stage_path(rel_dest_dir)
+      path = "#{path}/#{@component.subfolder}" if @component.subfolder
+      path
+    end
+  end
+end

data/lib/lono/bundler/exporter.rb

@@ -0,0 +1,50 @@
+module Lono::Bundler
+  class Exporter
+    include LB::Component::Concerns::PathConcern
+    include LB::Util::Logging
+
+    def initialize(options={})
+      @options = options
+    end
+
+    def run
+      purge_all
+      components.each do |component|
+        logger.info "Exporting #{component.export_path}"
+        purge(component)
+        export(component)
+      end
+    end
+
+    def components
+      components = lockfile.components
+      if LB.update_mode? && !@options[:components].empty?
+        components.select! { |component| @options[:components].include?(component.name) }
+      end
+      components
+    end
+
+    def export(component)
+      fetcher = Component::Fetcher.new(component).instance
+      fetcher.run
+      fetcher.switch_version(component.sha)
+      copy = Copy.new(component)
+      copy.component
+    end
+
+  private
+    def purge_all
+      return if LB.update_mode?
+      return unless LB.config.export_purge
+      FileUtils.rm_rf(LB.config.export_to)
+    end
+
+    def purge(component)
+      FileUtils.rm_rf(component.export_path)
+    end
+
+    def lockfile
+      Lockfile.instance
+    end
+  end
+end