terraspace-bundler 0.3.4 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 120ab895b524973b2a5a6367f76d8d1f8794cbc5957abae90ab11532833b8620
-  data.tar.gz: 55c69457dbd4fd1bdfce68b1432bcab70fea3d28805f28fc06961fe1c534c149
+  metadata.gz: c8245c4a3b7fd5b149298bdf0084bddeee9153ff1cd7016e7e13facb0f3bede2
+  data.tar.gz: 761d564519e6958842cc5386e7b84f4a2b2cc90e3801fc2c3845f752983241b2
 SHA512:
-  metadata.gz: 024b2f949df4eccfbdd5eaa7ce5b56a60ffc8f48f98044edd38408bd6f87a6f1e87f6b4fbcab54871884489c8742a7a92586eea27233e169cd2ea06c154bb8dc
-  data.tar.gz: 2270365d4fea228f1170601e2b02e54e9ccad230ed1dee044b9db134c0bd957f0e50fee715530ec9b1e976f174078e183cb0d598ca24eaf92ebe7f4ccc2d53e9
+  metadata.gz: d23b49ddabd983bf1927c62aabc2a9bf4165970a32b7325e41965bc804267297cf269741677582e4c0c4f7603e15dc21fef9d5370200b18adb2f369733b653ae
+  data.tar.gz: 38bf841d2c9d721b34930786c46567a47e881e3951153272eb7f9dea66e23e974994071e5514a4e92091db0fb19d787ce22893bfb514464ae967fcc04bb34959

data/CHANGELOG.md

@@ -3,6 +3,10 @@
 All notable changes to this project will be documented in this file.
 This project *loosely tries* to adhere to [Semantic Versioning](http://semver.org/), even before v1.0.
 
+## [0.4.0] - 2021-08-10
+- [#10](https://github.com/boltops-tools/terraspace-bundler/pull/10) terraspace bundle info: warning when no lockfile
+- [#12](https://github.com/boltops-tools/terraspace-bundler/pull/12) support for: private registry, s3, gcs, local
+
 ## [0.3.4] - 2021-04-15
 - [#11](https://github.com/boltops-tools/terraspace-bundler/pull/11) improve registry dectection and gitlab repository support
 

data/lib/terraspace_bundler/cli/purge_cache.rb

@@ -1,6 +1,6 @@
 class TerraspaceBundler::CLI
   class PurgeCache < Base
-    include TB::Mod::PathConcern
+    include TB::Mod::Concerns::PathConcern
     include TB::Util::Logging
     include TB::Util::Sure
 

data/lib/terraspace_bundler/dsl/syntax.rb

@@ -21,6 +21,10 @@ class TerraspaceBundler::Dsl
       config.stack_options.merge!(value)
     end
 
+    def clone_with(value)
+      config.clone_with = value
+    end
+
     def config
       TB.config
     end

data/lib/terraspace_bundler/exporter.rb

@@ -1,6 +1,6 @@
 module TerraspaceBundler
   class Exporter
-    include TB::Mod::PathConcern
+    include TB::Mod::Concerns::PathConcern
     include TB::Util::Logging
 
     def initialize(options={})
@@ -25,9 +25,9 @@ module TerraspaceBundler
     end
 
     def export(mod)
-      downloader = Mod::Downloader.new(mod)
-      downloader.run
-      downloader.switch_version(mod.sha)
+      fetcher = Mod::Fetcher.new(mod).instance
+      fetcher.run
+      fetcher.switch_version(mod.sha)
       copy = Copy.new(mod)
       copy.mod
       copy.stacks

data/lib/terraspace_bundler/exporter/base.rb

@@ -1,6 +1,6 @@
 class TerraspaceBundler::Exporter
   class Base
-    include TB::Mod::PathConcern
+    include TB::Mod::Concerns::PathConcern
     include TB::Util::Logging
   end
 end

data/lib/terraspace_bundler/exporter/copy.rb

@@ -7,6 +7,7 @@ class TerraspaceBundler::Exporter
     def mod
       FileUtils.rm_rf(mod_path)
       FileUtils.mkdir_p(File.dirname(mod_path))
+      logger.debug "Copy: cp -r #{src_path} #{mod_path}"
       FileUtils.cp_r(src_path, mod_path)
       FileUtils.rm_rf("#{mod_path}/.git")
     end
@@ -17,7 +18,7 @@ class TerraspaceBundler::Exporter
 
     # src path is from the stage area
     def src_path
-      path = stage_path(@mod.full_repo) # from PathConcern
+      path = stage_path(rel_dest_dir)
       path = "#{path}/#{@mod.subfolder}" if @mod.subfolder
       path
     end

data/lib/terraspace_bundler/extract.rb

@@ -0,0 +1,21 @@
+module TerraspaceBundler
+  class Extract
+    def self.extract(archive, dest)
+      FileUtils.rm_rf(dest)
+      FileUtils.mkdir_p(File.dirname(dest))
+
+      if archive.ends_with?('.tgz') || archive.ends_with?('.tar.gz')
+        Tar.extract(archive, dest)
+      elsif archive.ends_with?('.zip')
+        Zip.extract(archive, dest)
+      else
+        puts <<~EOL.color(:red)
+          ERROR: Unable to extract. Unsupported archive extension for:
+
+              #{archive}
+        EOL
+        exit 1
+      end
+    end
+  end
+end

data/lib/terraspace_bundler/extract/tar.rb

@@ -0,0 +1,34 @@
+require 'rubygems/package'
+require 'zlib'
+require 'fileutils'
+
+# Thanks: https://gist.github.com/ForeverZer0/2adbba36fd452738e7cca6a63aee2f30
+class TerraspaceBundler::Extract
+  class Tar
+    # tar_gz_archive = '/tmp/terraspace/bundler/stage/s3-us-west-2.amazonaws.com/demo-terraform-test/modules/example-module.tgz'
+    # destination = '/tmp/terraspace/where/extract/to'
+    def self.extract(tar_gz_archive, destination)
+      reader = Zlib::GzipReader.open(tar_gz_archive)
+      Gem::Package::TarReader.new(reader) do |tar|
+        dest = nil
+        tar.each do |entry|
+          dest ||= File.join destination, entry.full_name
+          if entry.directory?
+            FileUtils.rm_rf dest unless File.directory? dest
+            FileUtils.mkdir_p dest, :mode => entry.header.mode, :verbose => false
+          elsif entry.file?
+            FileUtils.rm_rf dest unless File.file? dest
+            File.open dest, "wb" do |f|
+              f.print entry.read
+            end
+            FileUtils.chmod entry.header.mode, dest, :verbose => false
+          elsif entry.header.typeflag == '2' #Symlink!
+            File.symlink entry.header.linkname, dest
+          end
+          dest = nil
+        end
+      end
+    end
+  end
+end
+

data/lib/terraspace_bundler/extract/zip.rb

@@ -0,0 +1,16 @@
+require 'rubygems'
+require 'zip'
+
+class TerraspaceBundler::Extract
+  class Zip
+    def self.extract(file, dest)
+      ::Zip::File.open(file) { |zip_file|
+        zip_file.each { |f|
+          f_path=File.join(dest, f.name)
+          FileUtils.mkdir_p(File.dirname(f_path))
+          zip_file.extract(f, f_path) unless File.exist?(f_path)
+        }
+      }
+    end
+  end
+end

data/lib/terraspace_bundler/info.rb

@@ -1,6 +1,13 @@
 module TerraspaceBundler
   class Info < TB::CLI::Base
     def run
+      file = TB.config.lockfile
+      unless File.exist?(file)
+        logger.info "No #{file} found".color(:red)
+        logger.info "Maybe run: terraspace bundle"
+        return
+      end
+      
       name = @options[:mod]
       found = lockfile.mods.find { |m| m.name == @options[:mod] }
       if found

data/lib/terraspace_bundler/mod.rb

@@ -1,9 +1,10 @@
 module TerraspaceBundler
   class Mod
-    extend PropsExtension
-    props :export_to, :name, :sha, :source, :subfolder, :type, :url
+    extend Props::Extension
+    props :export_to, :name, :sha, :source, :subfolder, :type, :url, :clone_with
 
-    include StackConcern
+    include Concerns::StackConcern
+    include Concerns::LocalConcern
 
     attr_reader :props, :version, :ref, :tag, :branch
     def initialize(props={})
@@ -19,7 +20,7 @@ module TerraspaceBundler
 
     # use url instead of source because for registry modules, the repo name is different
     def repo
-      url_words[-1]
+      url_words[-1].sub(/\.git$/,'')
     end
 
     # https://github.com/tongueroo/pet - 2nd to last word
@@ -34,9 +35,19 @@ module TerraspaceBundler
     end
 
     def latest_sha
-      downloader = Downloader.new(self)
-      downloader.run
-      downloader.sha
+      fetcher = Fetcher.new(self).instance
+      fetcher.run
+      fetcher.sha
+    end
+
+    def vcs_provider
+      if url.include?('http')
+        # "https://github.com/org/repo"  => github.com
+        url.match(%r{http[s]?://(.*?)/})[1]
+      else # git@
+        # "git@github.com:org/repo"      => github.com
+        url.match(%r{git@(.*?):})[1]
+      end
     end
 
   private

data/lib/terraspace_bundler/mod/concerns/local_concern.rb

@@ -0,0 +1,10 @@
+module TerraspaceBundler::Mod::Concerns
+  module LocalConcern
+    def local?
+      source.starts_with?("/")  ||
+      source.starts_with?(".")  ||
+      source.starts_with?("..") ||
+      source.starts_with?("~")
+    end
+  end
+end

data/lib/terraspace_bundler/mod/concerns/notation_concern.rb

@@ -0,0 +1,42 @@
+require 'uri'
+
+module TerraspaceBundler::Mod::Concerns
+  module NotationConcern
+    def remove_notations(source)
+      remove_subfolder_notation(remove_ref_notation(source))
+    end
+
+    def remove_ref_notation(source)
+      source.sub(/\?.*/,'')
+    end
+
+    def remove_subfolder_notation(source)
+      parts = clean_notation(source).split('//')
+      if parts.size == 2 # has subfolder
+        source.split('//')[0..-2].join('//') # remove only subfolder, keep rest of original source
+      else
+        source
+      end
+    end
+
+    def subfolder(source)
+      parts = clean_notation(source).split('//')
+      if parts.size == 2 # has subfolder
+        remove_ref_notation(parts.last)
+      end
+    end
+
+    def ref(source)
+      url = clean_notation(source)
+      uri = URI(url)
+      if uri.query
+        params = URI::decode_www_form(uri.query).to_h # if you are in 2.1 or later version of Ruby
+        params['ref']
+      end
+    end
+
+    def clean_notation(source)
+      source.sub(/.*::/,'').sub(%r{http[s?]://},'').sub(%r{git@(.*?):},'') # also remove git@ notation
+    end
+  end
+end

data/lib/terraspace_bundler/mod/concerns/path_concern.rb

@@ -0,0 +1,90 @@
+module TerraspaceBundler::Mod::Concerns
+  module PathConcern
+    def setup_tmp
+      FileUtils.mkdir_p(cache_root)
+      FileUtils.mkdir_p(stage_root)
+    end
+
+    def tmp_root
+      "/tmp/terraspace/bundler"
+    end
+
+    def cache_root
+      "#{tmp_root}/cache"
+    end
+
+    def stage_root
+      "#{tmp_root}/stage"
+    end
+
+    def cache_path(name)
+      [cache_root, parent_stage_folder, name].compact.join('/')
+    end
+
+    def stage_path(name)
+      [stage_root, parent_stage_folder, name].compact.join('/')
+    end
+
+    # Fetcher: Downloader/Local copies to a slightly different folder.
+    # Also, Copy will use this and reference same method so it's consistent.
+    def rel_dest_dir
+      case @mod.type
+      when 'local'
+        @mod.name      # example-module
+      when 's3'
+        path = type_path # https://s3-us-west-2.amazonaws.com/demo-terraform-test/example-module.zip
+        remove_ext(path) # demo-terraform-test/modules/example-module
+      when 'gcs'
+        path = type_path # https://www.googleapis.com/storage/v1/BUCKET_NAME/PATH/TO/module.zip
+        path.sub!(%r{storage/v\d+/},'')
+        remove_ext(path) # terraform-example-modules/modules/example-module
+      when 'http'
+        path = type_path # https://www.googleapis.com/storage/v1/BUCKET_NAME/PATH/TO/module.zip
+        remove_ext(path) # terraform-example-modules/modules/example-module
+      when -> (_) { @mod.source.include?('git::') }
+        @mod.name      # example-module
+      else # inferred git, registry
+        @mod.full_repo #  tongueroo/example-module
+      end
+    end
+
+    def parent_stage_folder
+      case @mod.type
+      when 'local'
+        'local'
+      when 'http'
+        'http'
+      else # gcs, s3, git, registry
+        @mod.vcs_provider
+      end
+    end
+
+    def type_path
+      source = @mod.source
+      url = source.sub("#{@mod.type}::",'')
+      uri = URI(url)
+      uri.path.sub('/','')   # removing leading slash to bucket name is the first thing
+         .sub(%r{//(.*)},'') # remove subfolder
+    end
+
+    def remove_ext(path)
+      ext = File.extname(path)
+      path.sub(ext,'')
+    end
+
+    def get_bucket_key(path)
+      bucket, *rest = path.split('/')
+      key = rest.join('/')
+      [bucket, key]
+    end
+
+    def mod_path
+      get_mod_path(@mod)
+    end
+
+    def get_mod_path(mod)
+      export_to = mod.export_to || TB.config.export_to
+      "#{export_to}/#{mod.name}"
+    end
+  end
+end

data/lib/terraspace_bundler/mod/{stack_concern.rb → concerns/stack_concern.rb}

@@ -1,4 +1,4 @@
-class TerraspaceBundler::Mod
+module TerraspaceBundler::Mod::Concerns
   module StackConcern
     def stacks
       stacks = @props[:stacks] || @props[:stack]

data/lib/terraspace_bundler/mod/fetcher.rb

@@ -0,0 +1,18 @@
+# Delegates to:
+#
+#   1. Local
+#   2. Git
+#
+class TerraspaceBundler::Mod
+  class Fetcher
+    def initialize(mod)
+      @mod = mod
+    end
+
+    def instance
+      type = @mod.type == "registry" ? "git" : @mod.type
+      klass = "TerraspaceBundler::Mod::Fetcher::#{type.camelize}".constantize
+      klass.new(@mod) # IE: Local.new(@mod), Git.new(@mod), S3.new(@mod), etc
+    end
+  end
+end

data/lib/terraspace_bundler/mod/fetcher/base.rb

@@ -0,0 +1,27 @@
+# Interface of subclasses should implement
+#
+#   run
+#   switch_version(mod.sha)
+#   sha
+#
+class TerraspaceBundler::Mod::Fetcher
+  class Base
+    include TB::Util::Git
+    include TB::Util::Logging
+    include TB::Mod::Concerns::PathConcern
+
+    attr_reader :sha # returns nil for Local
+    def initialize(mod)
+      @mod = mod
+    end
+
+    def switch_version(*)
+      # noop
+    end
+
+    def extract(archive, dest)
+      TerraspaceBundler::Extract.extract(archive, dest)
+    end
+
+  end
+end

data/lib/terraspace_bundler/mod/fetcher/gcs.rb

@@ -0,0 +1,48 @@
+require "google/cloud/storage"
+
+class TerraspaceBundler::Mod::Fetcher
+  class Gcs < Base
+    extend Memoist
+
+    def run
+      bucket, key, path = gcs_info
+      download(bucket, key, path)
+    end
+
+    def download(bucket_name, key, path)
+      # Download to cache area
+      bucket = storage.bucket(bucket_name)
+      unless bucket
+        logger.error "ERROR: bucket #{bucket_name} does not exist".color(:red)
+        exit 1
+      end
+      file = bucket.file(key)
+      unless file
+        logger.error "ERROR: Unable to find gs://#{bucket_name}/#{key}".color(:red)
+        exit 1
+      end
+
+      archive = cache_path(path) # temporary path
+      logger.debug "Gcs save archive to #{archive}"
+      FileUtils.mkdir_p(File.dirname(archive))
+      file.download(archive)
+
+      # Save to stage
+      dest = stage_path(rel_dest_dir)
+      extract(archive, dest)
+    end
+
+  private
+    def gcs_info
+      path = type_path
+      path.sub!(%r{storage/v\d+/},'')
+      bucket, key = get_bucket_key(path)
+      [bucket, key, path]
+    end
+
+    def storage
+      Google::Cloud::Storage.new
+    end
+    memoize :storage
+  end
+end