terraspace-bundler 0.3.2 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3a02d9119392f13a0a31079c756f565f1de0e4d200a3af9f84c732f4656ed6fc
-  data.tar.gz: d9e07212f6efce634f6836bb8ac2a1dd184dc5a34562c24252c8e69c27f4889c
+  metadata.gz: f59f992f4e07582208760a29beac2bd1a507164e6cfb19fe4fca27ac1f9d04dc
+  data.tar.gz: df2b9be3142d30891cb959cafcd5376987e92192b24a85e274606fe761bb1742
 SHA512:
-  metadata.gz: 6e79f9d42d0ddc7856877ce4d1ce0dd2452eddb883e68694b28bc776512a56bcf829747bd589a66a28bf9687dca590404ff6cc8906df57c96a8dc62c40d12571
-  data.tar.gz: 1915a7ec28c0cdce027b5f5ebcee59924853d4ad3ad8de2cc6e0db5ab5ff7f6d4d2260881e9df42cbeddc226d5574793701533647d07c86f576c04c4b3dcc7dc
+  metadata.gz: 73a7f1ca1642c6fe6707b30ac429cc8ca958e37fb86393bf54fea2345ccc0d3cfee2a2a9e7b63dbcb936338e774d1feed95a3276bdbd9b906109671313f61de8
+  data.tar.gz: 02f65ae63819e83d994de88c57c6c32c078d9627a2e6bebea9711c74d47c48f18186d1bd734989618a17f096a0459586649a9a1b8615e30a73f29341200dbc92

data/CHANGELOG.md

@@ -3,6 +3,20 @@
 All notable changes to this project will be documented in this file.
 This project *loosely tries* to adhere to [Semantic Versioning](http://semver.org/), even before v1.0.
 
+## [0.4.1] - 2021-09-30
+- [#13](https://github.com/boltops-tools/terraspace-bundler/pull/13) fix stacks Array option
+
+## [0.4.0] - 2021-08-10
+- [#10](https://github.com/boltops-tools/terraspace-bundler/pull/10) terraspace bundle info: warning when no lockfile
+- [#12](https://github.com/boltops-tools/terraspace-bundler/pull/12) support for: private registry, s3, gcs, local
+
+## [0.3.4] - 2021-04-15
+- [#11](https://github.com/boltops-tools/terraspace-bundler/pull/11) improve registry dectection and gitlab repository support
+
+## [0.3.3] - 2021-02-24
+- [#8](https://github.com/boltops-tools/terraspace-bundler/pull/8) fix https sources
+- [#9](https://github.com/boltops-tools/terraspace-bundler/pull/9) Git checkout retry with v
+
 ## [0.3.2]
 - #4 only purge and export explicit mods
 - are you sure prompt for purge_cache

data/lib/terraspace_bundler/cli/bundle.rb

@@ -37,7 +37,7 @@ class TerraspaceBundler::CLI
     terrafile_option.call
     def update(*mods)
       TB.update_mode = true
-      TB::Runner.new(options.merge(mods: mods, mode: "update")).run
+      TB::Runner.new(options.merge(mods: mods)).run
     end
   end
 end

data/lib/terraspace_bundler/cli/purge_cache.rb

@@ -1,6 +1,6 @@
 class TerraspaceBundler::CLI
   class PurgeCache < Base
-    include TB::Mod::PathConcern
+    include TB::Mod::Concerns::PathConcern
     include TB::Util::Logging
     include TB::Util::Sure
 

data/lib/terraspace_bundler/dsl/syntax.rb

@@ -21,6 +21,10 @@ class TerraspaceBundler::Dsl
       config.stack_options.merge!(value)
     end
 
+    def clone_with(value)
+      config.clone_with = value
+    end
+
     def config
       TB.config
     end

data/lib/terraspace_bundler/exporter/base.rb

@@ -1,6 +1,6 @@
 class TerraspaceBundler::Exporter
   class Base
-    include TB::Mod::PathConcern
+    include TB::Mod::Concerns::PathConcern
     include TB::Util::Logging
   end
 end

data/lib/terraspace_bundler/exporter/copy.rb

@@ -7,6 +7,7 @@ class TerraspaceBundler::Exporter
     def mod
       FileUtils.rm_rf(mod_path)
       FileUtils.mkdir_p(File.dirname(mod_path))
+      logger.debug "Copy: cp -r #{src_path} #{mod_path}"
       FileUtils.cp_r(src_path, mod_path)
       FileUtils.rm_rf("#{mod_path}/.git")
     end
@@ -17,7 +18,7 @@ class TerraspaceBundler::Exporter
 
     # src path is from the stage area
     def src_path
-      path = stage_path(@mod.full_repo) # from PathConcern
+      path = stage_path(rel_dest_dir)
       path = "#{path}/#{@mod.subfolder}" if @mod.subfolder
       path
     end

data/lib/terraspace_bundler/exporter.rb

@@ -1,6 +1,6 @@
 module TerraspaceBundler
   class Exporter
-    include TB::Mod::PathConcern
+    include TB::Mod::Concerns::PathConcern
     include TB::Util::Logging
 
     def initialize(options={})
@@ -25,9 +25,9 @@ module TerraspaceBundler
     end
 
     def export(mod)
-      downloader = Mod::Downloader.new(mod)
-      downloader.run
-      downloader.switch_version(mod.sha)
+      fetcher = Mod::Fetcher.new(mod).instance
+      fetcher.run
+      fetcher.switch_version(mod.sha)
       copy = Copy.new(mod)
       copy.mod
       copy.stacks

data/lib/terraspace_bundler/extract/tar.rb

@@ -0,0 +1,34 @@
+require 'rubygems/package'
+require 'zlib'
+require 'fileutils'
+
+# Thanks: https://gist.github.com/ForeverZer0/2adbba36fd452738e7cca6a63aee2f30
+class TerraspaceBundler::Extract
+  class Tar
+    # tar_gz_archive = '/tmp/terraspace/bundler/stage/s3-us-west-2.amazonaws.com/demo-terraform-test/modules/example-module.tgz'
+    # destination = '/tmp/terraspace/where/extract/to'
+    def self.extract(tar_gz_archive, destination)
+      reader = Zlib::GzipReader.open(tar_gz_archive)
+      Gem::Package::TarReader.new(reader) do |tar|
+        dest = nil
+        tar.each do |entry|
+          dest ||= File.join destination, entry.full_name
+          if entry.directory?
+            FileUtils.rm_rf dest unless File.directory? dest
+            FileUtils.mkdir_p dest, :mode => entry.header.mode, :verbose => false
+          elsif entry.file?
+            FileUtils.rm_rf dest unless File.file? dest
+            File.open dest, "wb" do |f|
+              f.print entry.read
+            end
+            FileUtils.chmod entry.header.mode, dest, :verbose => false
+          elsif entry.header.typeflag == '2' #Symlink!
+            File.symlink entry.header.linkname, dest
+          end
+          dest = nil
+        end
+      end
+    end
+  end
+end
+

data/lib/terraspace_bundler/extract/zip.rb

@@ -0,0 +1,16 @@
+require 'rubygems'
+require 'zip'
+
+class TerraspaceBundler::Extract
+  class Zip
+    def self.extract(file, dest)
+      ::Zip::File.open(file) { |zip_file|
+        zip_file.each { |f|
+          f_path=File.join(dest, f.name)
+          FileUtils.mkdir_p(File.dirname(f_path))
+          zip_file.extract(f, f_path) unless File.exist?(f_path)
+        }
+      }
+    end
+  end
+end

data/lib/terraspace_bundler/extract.rb

@@ -0,0 +1,21 @@
+module TerraspaceBundler
+  class Extract
+    def self.extract(archive, dest)
+      FileUtils.rm_rf(dest)
+      FileUtils.mkdir_p(File.dirname(dest))
+
+      if archive.ends_with?('.tgz') || archive.ends_with?('.tar.gz')
+        Tar.extract(archive, dest)
+      elsif archive.ends_with?('.zip')
+        Zip.extract(archive, dest)
+      else
+        puts <<~EOL.color(:red)
+          ERROR: Unable to extract. Unsupported archive extension for:
+
+              #{archive}
+        EOL
+        exit 1
+      end
+    end
+  end
+end

data/lib/terraspace_bundler/info.rb

@@ -1,6 +1,13 @@
 module TerraspaceBundler
   class Info < TB::CLI::Base
     def run
+      file = TB.config.lockfile
+      unless File.exist?(file)
+        logger.info "No #{file} found".color(:red)
+        logger.info "Maybe run: terraspace bundle"
+        return
+      end
+      
       name = @options[:mod]
       found = lockfile.mods.find { |m| m.name == @options[:mod] }
       if found

data/lib/terraspace_bundler/mod/concerns/local_concern.rb

@@ -0,0 +1,10 @@
+module TerraspaceBundler::Mod::Concerns
+  module LocalConcern
+    def local?
+      source.starts_with?("/")  ||
+      source.starts_with?(".")  ||
+      source.starts_with?("..") ||
+      source.starts_with?("~")
+    end
+  end
+end

data/lib/terraspace_bundler/mod/concerns/notation_concern.rb

@@ -0,0 +1,42 @@
+require 'uri'
+
+module TerraspaceBundler::Mod::Concerns
+  module NotationConcern
+    def remove_notations(source)
+      remove_subfolder_notation(remove_ref_notation(source))
+    end
+
+    def remove_ref_notation(source)
+      source.sub(/\?.*/,'')
+    end
+
+    def remove_subfolder_notation(source)
+      parts = clean_notation(source).split('//')
+      if parts.size == 2 # has subfolder
+        source.split('//')[0..-2].join('//') # remove only subfolder, keep rest of original source
+      else
+        source
+      end
+    end
+
+    def subfolder(source)
+      parts = clean_notation(source).split('//')
+      if parts.size == 2 # has subfolder
+        remove_ref_notation(parts.last)
+      end
+    end
+
+    def ref(source)
+      url = clean_notation(source)
+      uri = URI(url)
+      if uri.query
+        params = URI::decode_www_form(uri.query).to_h # if you are in 2.1 or later version of Ruby
+        params['ref']
+      end
+    end
+
+    def clean_notation(source)
+      source.sub(/.*::/,'').sub(%r{http[s?]://},'').sub(%r{git@(.*?):},'') # also remove git@ notation
+    end
+  end
+end

data/lib/terraspace_bundler/mod/concerns/path_concern.rb

@@ -0,0 +1,90 @@
+module TerraspaceBundler::Mod::Concerns
+  module PathConcern
+    def setup_tmp
+      FileUtils.mkdir_p(cache_root)
+      FileUtils.mkdir_p(stage_root)
+    end
+
+    def tmp_root
+      "/tmp/terraspace/bundler"
+    end
+
+    def cache_root
+      "#{tmp_root}/cache"
+    end
+
+    def stage_root
+      "#{tmp_root}/stage"
+    end
+
+    def cache_path(name)
+      [cache_root, parent_stage_folder, name].compact.join('/')
+    end
+
+    def stage_path(name)
+      [stage_root, parent_stage_folder, name].compact.join('/')
+    end
+
+    # Fetcher: Downloader/Local copies to a slightly different folder.
+    # Also, Copy will use this and reference same method so it's consistent.
+    def rel_dest_dir
+      case @mod.type
+      when 'local'
+        @mod.name      # example-module
+      when 's3'
+        path = type_path # https://s3-us-west-2.amazonaws.com/demo-terraform-test/example-module.zip
+        remove_ext(path) # demo-terraform-test/modules/example-module
+      when 'gcs'
+        path = type_path # https://www.googleapis.com/storage/v1/BUCKET_NAME/PATH/TO/module.zip
+        path.sub!(%r{storage/v\d+/},'')
+        remove_ext(path) # terraform-example-modules/modules/example-module
+      when 'http'
+        path = type_path # https://www.googleapis.com/storage/v1/BUCKET_NAME/PATH/TO/module.zip
+        remove_ext(path) # terraform-example-modules/modules/example-module
+      when -> (_) { @mod.source.include?('git::') }
+        @mod.name      # example-module
+      else # inferred git, registry
+        @mod.full_repo #  tongueroo/example-module
+      end
+    end
+
+    def parent_stage_folder
+      case @mod.type
+      when 'local'
+        'local'
+      when 'http'
+        'http'
+      else # gcs, s3, git, registry
+        @mod.vcs_provider
+      end
+    end
+
+    def type_path
+      source = @mod.source
+      url = source.sub("#{@mod.type}::",'')
+      uri = URI(url)
+      uri.path.sub('/','')   # removing leading slash to bucket name is the first thing
+         .sub(%r{//(.*)},'') # remove subfolder
+    end
+
+    def remove_ext(path)
+      ext = File.extname(path)
+      path.sub(ext,'')
+    end
+
+    def get_bucket_key(path)
+      bucket, *rest = path.split('/')
+      key = rest.join('/')
+      [bucket, key]
+    end
+
+    def mod_path
+      get_mod_path(@mod)
+    end
+
+    def get_mod_path(mod)
+      export_to = mod.export_to || TB.config.export_to
+      "#{export_to}/#{mod.name}"
+    end
+  end
+end

data/lib/terraspace_bundler/mod/{stack_concern.rb → concerns/stack_concern.rb}

@@ -1,10 +1,12 @@
-class TerraspaceBundler::Mod
+module TerraspaceBundler::Mod::Concerns
   module StackConcern
     def stacks
       stacks = @props[:stacks] || @props[:stack]
       return unless stacks
       if all_stacks?(stacks)
         stacks = all_stacks
+      elsif stacks.is_a?(Array)
+        stacks = array_stacks(stacks)
       end
       normalize_stacks(stacks)
     end
@@ -28,6 +30,19 @@ class TerraspaceBundler::Mod
       end
     end
 
+    def array_stacks(stacks)
+      stacks.map do |example|
+        if example.is_a?(Hash)
+          example
+        else
+          {
+            name: example,
+            src: example,
+          }
+        end
+      end
+    end
+
     # Normalizes stack options to an Array of Hashes or just a single Hash
     def normalize_stacks(option)
       defaults = TB.config.stack_options.dup

data/lib/terraspace_bundler/mod/fetcher/base.rb

@@ -0,0 +1,27 @@
+# Interface of subclasses should implement
+#
+#   run
+#   switch_version(mod.sha)
+#   sha
+#
+class TerraspaceBundler::Mod::Fetcher
+  class Base
+    include TB::Util::Git
+    include TB::Util::Logging
+    include TB::Mod::Concerns::PathConcern
+
+    attr_reader :sha # returns nil for Local
+    def initialize(mod)
+      @mod = mod
+    end
+
+    def switch_version(*)
+      # noop
+    end
+
+    def extract(archive, dest)
+      TerraspaceBundler::Extract.extract(archive, dest)
+    end
+
+  end
+end

data/lib/terraspace_bundler/mod/fetcher/gcs.rb

@@ -0,0 +1,48 @@
+require "google/cloud/storage"
+
+class TerraspaceBundler::Mod::Fetcher
+  class Gcs < Base
+    extend Memoist
+
+    def run
+      bucket, key, path = gcs_info
+      download(bucket, key, path)
+    end
+
+    def download(bucket_name, key, path)
+      # Download to cache area
+      bucket = storage.bucket(bucket_name)
+      unless bucket
+        logger.error "ERROR: bucket #{bucket_name} does not exist".color(:red)
+        exit 1
+      end
+      file = bucket.file(key)
+      unless file
+        logger.error "ERROR: Unable to find gs://#{bucket_name}/#{key}".color(:red)
+        exit 1
+      end
+
+      archive = cache_path(path) # temporary path
+      logger.debug "Gcs save archive to #{archive}"
+      FileUtils.mkdir_p(File.dirname(archive))
+      file.download(archive)
+
+      # Save to stage
+      dest = stage_path(rel_dest_dir)
+      extract(archive, dest)
+    end
+
+  private
+    def gcs_info
+      path = type_path
+      path.sub!(%r{storage/v\d+/},'')
+      bucket, key = get_bucket_key(path)
+      [bucket, key, path]
+    end
+
+    def storage
+      Google::Cloud::Storage.new
+    end
+    memoize :storage
+  end
+end

data/lib/terraspace_bundler/mod/{downloader.rb → fetcher/git.rb}

@@ -1,24 +1,18 @@
-class TerraspaceBundler::Mod
-  class Downloader
+class TerraspaceBundler::Mod::Fetcher
+  class Git < Base
     extend Memoist
-    include TB::Util::Git
-    include TB::Util::Logging
-    include TB::Mod::PathConcern
-
-    attr_reader :sha
-    def initialize(mod)
-      @mod = mod
-    end
 
     def run
       setup_tmp
-      org_path = "#{cache_root}/#{@mod.org}"
+      org_path = cache_path(@mod.org)
       FileUtils.mkdir_p(org_path)
 
       Dir.chdir(org_path) do
+        logger.debug "Current root dir: #{org_path}"
         clone unless File.exist?(@mod.repo)
 
         Dir.chdir(@mod.repo) do
+          logger.debug "Change dir: #{@mod.repo}"
           git "pull"
           git "submodule update --init"
           stage
@@ -28,7 +22,8 @@ class TerraspaceBundler::Mod
     memoize :run
 
     def clone
-      sh "git clone #{@mod.url}"
+      command = ["git clone", ENV['TB_GIT_CLONE_ARGS'], @mod.url].compact.join(' ')
+      sh command
     rescue TB::GitError => e
       logger.error "ERROR: #{e.message}".color(:red)
       exit 1
@@ -55,8 +50,7 @@ class TerraspaceBundler::Mod
     end
 
     def switch_version(version)
-      stage_path = stage_path("#{@mod.org}/#{@mod.repo}")
-      logger.debug "Within: #{stage_path}"
+      stage_path = stage_path(rel_dest_dir)
       Dir.chdir(stage_path) do
         git "checkout #{version}"
         @sha = git("rev-parse HEAD").strip
@@ -64,8 +58,8 @@ class TerraspaceBundler::Mod
     end
 
     def copy_to_stage
-      cache_path = cache_path("#{@mod.org}/#{@mod.repo}")
-      stage_path = stage_path("#{@mod.org}/#{@mod.repo}")
+      cache_path = cache_path(rel_dest_dir)
+      stage_path = stage_path(rel_dest_dir)
       FileUtils.rm_rf(stage_path)
       FileUtils.mkdir_p(File.dirname(stage_path))
       FileUtils.cp_r(cache_path, stage_path)

data/lib/terraspace_bundler/mod/fetcher/local.rb

@@ -0,0 +1,15 @@
+class TerraspaceBundler::Mod::Fetcher
+  class Local < Base
+    def run
+      stage_path = stage_path(rel_dest_dir)
+      source = @mod.source
+      src = source.sub(/^~/, ENV['HOME']) # allow ~/ notation
+      FileUtils.rm_rf(stage_path)
+      FileUtils.mkdir_p(File.dirname(stage_path))
+      logger.debug "Local: cp -r #{src} #{stage_path}"
+      # copy from stage area to vendor/modules/NAME
+      # IE: cp -r /tmp/terraspace/bundler/stage/local/local1 vendor/modules/local1
+      FileUtils.cp_r(src, stage_path)
+    end
+  end
+end

data/lib/terraspace_bundler/mod/fetcher/s3.rb

@@ -0,0 +1,66 @@
+require 'aws-sdk-s3'
+require 'uri'
+
+class TerraspaceBundler::Mod::Fetcher
+  class S3 < Base
+    extend Memoist
+
+    def run
+      region, bucket, key, path = s3_info
+      download(region, bucket, key, path)
+    end
+
+    def download(region, bucket, key, path)
+      # Download to cache area
+      response_target = cache_path(path) # temporary path
+
+      unless File.exist?(response_target)
+        logger.debug "S3 save archive to #{response_target}".color(:yellow)
+        FileUtils.mkdir_p(File.dirname(response_target))
+        s3_get(region, response_target, bucket, key)
+      end
+
+      # Save to stage
+      dest = stage_path(rel_dest_dir)
+      extract(response_target, dest)
+    end
+
+  private
+    def s3_get(region, response_target, bucket, key)
+      s3(region).get_object(
+        response_target: response_target,
+        bucket: bucket,
+        key: key,
+      )
+    rescue Aws::S3::Errors::NoSuchBucket, Aws::S3::Errors::NoSuchKey => e
+      logger.error(<<~EOL.color(:red))
+        ERROR: #{e.class} #{e.message}
+
+            bucket: #{bucket}
+            key: #{key}
+
+      EOL
+      exit 1
+    end
+
+    def s3_info
+      path = type_path
+      bucket, key = get_bucket_key(path)
+
+      url = @mod.source.sub('s3::','')
+      uri = URI(url)
+      region = if uri.host == 'https://s3.amazonaws.com'
+                 'us-east-1'
+               else
+                 uri.host.match(/s3-(.*)\.amazonaws/)[1]
+               end
+
+      [region, bucket, key, path]
+    end
+
+    def s3(region)
+      Aws::S3::Client.new(region: region)
+    end
+    memoize :s3
+  end
+end

data/lib/terraspace_bundler/mod/fetcher.rb

@@ -0,0 +1,18 @@
+# Delegates to:
+#
+#   1. Local
+#   2. Git
+#
+class TerraspaceBundler::Mod
+  class Fetcher
+    def initialize(mod)
+      @mod = mod
+    end
+
+    def instance
+      type = @mod.type == "registry" ? "git" : @mod.type
+      klass = "TerraspaceBundler::Mod::Fetcher::#{type.camelize}".constantize
+      klass.new(@mod) # IE: Local.new(@mod), Git.new(@mod), S3.new(@mod), etc
+    end
+  end
+end

data/lib/terraspace_bundler/mod/http/concern.rb

@@ -0,0 +1,45 @@
+require 'net/http'
+require 'open-uri'
+
+module TerraspaceBundler::Mod::Http
+  module Concern
+    include TB::Util::Logging
+
+    def http_request(url, auth_domain: nil)
+      uri = URI(url)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = uri.scheme == "https"
+      # Total time will be 40s = 20 x 2
+      http.max_retries = 1 # Default is already 1, just  being explicit
+      http.read_timeout = 20 # Sites that dont return in 20 seconds are considered down
+      request = Net::HTTP::Get.new(uri)
+
+      if auth_domain
+        path = "#{ENV['HOME']}/.terraform.d/credentials.tfrc.json"
+        if File.exist?(path)
+          data = JSON.load(IO.read(path))
+          token = data['credentials'][auth_domain]['token']
+          request.add_field 'Authorization', "Bearer #{token}"
+        else
+          auth_error_exit!
+        end
+      end
+
+      begin
+         http.request(request) # response
+      rescue Net::OpenTimeout => e # internal ELB but VPC is not configured for Lambda function
+        http_request_error_message(e)
+      rescue Exception => e
+        # Net::ReadTimeout - too slow
+        # Errno::ECONNREFUSED - completely down
+        # SocketError - improper url "dsfjsl" instead of example.com
+        http_request_error_message(e)
+      end
+    end
+
+    def http_request_error_message(e)
+      logger.info "ERROR: #{e.message}\n#{e.message}".color(:red)
+      exit 1
+    end
+  end
+end