terraspace-bundler 0.1.0

data/lib/terraspace_bundler/helper/git.rb

@@ -0,0 +1,21 @@
+module TerraspaceBundler::Helper
+  module Git
+    include TB::Logging
+
+    def sh(command)
+      command = "#{command} 2>&1" # always need output for the sha
+      logger.debug "=> #{command}"
+      out = `#{command}`
+      unless $?.success?
+        logger.error "ERROR: running #{command}"
+        logger.error out
+        exit $?.exitstatus
+      end
+      out
+    end
+
+    def git(command)
+      sh("git #{command}")
+    end
+  end
+end

data/lib/terraspace_bundler/installer.rb

@@ -0,0 +1,45 @@
+module TerraspaceBundler
+  class Installer < CLI::Base
+    extend Memoist
+
+    def initialize(options={})
+      super
+      @download = options[:download].nil? ? true : options[:download]
+    end
+
+    def run
+      Setup.new(@options).setup!
+      download
+      export
+      logger.info "Modules saved to #{TB.config.export_path}"
+    end
+
+    def download
+      return unless @download
+
+      if File.exist?(TB.config.lockfile)
+        puts "Bundling modules from #{TB.config.lockfile}..."
+      else
+        Updater.new(@options).run(without_install: true) # creates Terrafile.lock
+      end
+
+      Syncer.new(mods).run
+    end
+
+    def export
+      locked_mods.each(&:export)
+    end
+
+    def mods
+      locked_mods.map(&:to_mod)
+    end
+    memoize :mods
+
+    def locked_mods
+      data = YAML.load_file(TB.config.lockfile).deep_symbolize_keys
+      data.map do |name, info|
+        Mod::Locked.new(name, info)
+      end
+    end
+  end
+end

data/lib/terraspace_bundler/logger.rb

@@ -0,0 +1,26 @@
+require 'logger'
+
+module TerraspaceBundler
+  class Logger < ::Logger
+    # Only need to override the add method as the other calls all lead to it.
+    def add(severity, message = nil, progname = nil)
+      # Taken from Logger#add source
+      # https://ruby-doc.org/stdlib-2.5.1/libdoc/logger/rdoc/Logger.html#method-i-add
+      if message.nil?
+        if block_given?
+          message = yield
+        else
+          message = progname
+          progname = @progname
+        end
+      end
+
+      super # original logic
+    end
+
+    # plain formatting
+    def format_message(severity, timestamp, progname, msg)
+      "#{msg}\n"
+    end
+  end
+end

data/lib/terraspace_bundler/logging.rb

@@ -0,0 +1,7 @@
+module TerraspaceBundler
+  module Logging
+    def logger
+      TerraspaceBundler.logger
+    end
+  end
+end

data/lib/terraspace_bundler/mod.rb

@@ -0,0 +1,85 @@
+module TerraspaceBundler
+  class Mod
+    extend Memoist
+    include TB::Util::Registry
+
+    attr_reader :branch, :ref, :tag, :sha, :source
+    def initialize(data={}, global={})
+      @data, @global = data, global
+      @args = data[:args]
+      @options = data[:options]
+
+      # support variety of options, prefer version
+      @version = @options[:version]
+      @branch = @options[:branch]
+      @ref = @options[:ref]
+      @tag = @options[:tag]
+    end
+
+    def source
+      if @options[:source].split('/').size == 1
+        "#{org}/#{@options[:source]}"
+      else
+        @options[:source]
+      end
+    end
+
+    def full_org
+      normalize_git_url(org)
+    end
+
+    def org
+      obtain_org(@options[:source], @global[:org])
+    end
+
+    def url
+      normalize_url(source)
+    end
+
+    def normalize_url(source)
+      return registry_github if registry?(source)
+      normalize_git_url(source)
+    end
+
+    def path
+      @options[:path]
+    end
+
+    def name
+      @args.first
+    end
+
+    def version
+      @version || @ref || @tag || @branch
+    end
+
+    def checkout_version
+      v = version
+      v = "v#{v}" if registry?(source) && @version && !v.starts_with?("v")
+      v
+    end
+
+    def sync
+      sync = Sync.new(self, url)
+      sync.run
+      @sha = sync.sha
+    end
+    memoize :sync
+
+    def normalize_git_url(s)
+      if git_url?(s)
+        s
+      else
+        "#{base_url}#{s}"
+      end
+    end
+
+    def base_url
+      @global[:base_url] || "git@github.com:"
+    end
+
+    def git_url?(s)
+      s.include?("http") || s.include?("git@")
+    end
+  end
+end

data/lib/terraspace_bundler/mod/export.rb

@@ -0,0 +1,23 @@
+class TerraspaceBundler::Mod
+  class Export
+    include TmpPaths
+
+    def initialize(locked_mod)
+      @locked_mod = locked_mod
+    end
+
+    def run
+      stage_path = stage_path(@locked_mod.full_repo)
+      stage_path = "#{stage_path}/#{@locked_mod.path}" if @locked_mod.path
+      export_path = export_path(@locked_mod.name)
+      FileUtils.rm_rf(export_path)
+      FileUtils.mkdir_p(File.dirname(export_path))
+      FileUtils.cp_r(stage_path, export_path)
+      FileUtils.rm_rf("#{export_path}/.git")
+    end
+
+    def export_path(repo)
+      "#{TB.config.export_path}/#{repo}"
+    end
+  end
+end

data/lib/terraspace_bundler/mod/locked.rb

@@ -0,0 +1,48 @@
+class TerraspaceBundler::Mod
+  class Locked
+    include TB::Util::Registry
+
+    attr_reader :name, :path, :source, :version
+    def initialize(name, info)
+      @name, @info = name.to_s, info
+
+      @path = @info[:path]
+      @version = @info[:version]
+    end
+
+    def source
+      @info[:source]
+    end
+
+    def org
+      obtain_org(source)
+    end
+
+    def repo
+      s = registry?(source) ? registry_github : source
+      File.basename(s)
+    end
+
+    def full_repo
+      "#{org}/#{repo}"
+    end
+
+    def to_mod
+      # copy all props in Terrafile.lock
+      options = @info.clone
+      # add org
+      options.merge!(org: @org)
+
+      data = {
+        args: @name,
+        options: options,
+      }
+      TB::Mod.new(data) # no need for global, info in the lockfile is enough
+    end
+
+    def export
+      export = Export.new(self)
+      export.run
+    end
+  end
+end

data/lib/terraspace_bundler/mod/registry.rb

@@ -0,0 +1,72 @@
+require 'net/http'
+require 'open-uri'
+
+class TerraspaceBundler::Mod
+  class Registry
+    def initialize(mod)
+      @mod = mod
+    end
+
+    # Terrafile example
+    #
+    #      mod "sg", source: "terraform-aws-modules/security-group/aws", version: "3.10.0"
+    #
+    # Resources: https://www.terraform.io/docs/registry/api.html
+    #
+    # Latest version:
+    #
+    #     https://registry.terraform.io/v1/modules/terraform-aws-modules/sqs/aws/2.1.0/download
+    #
+    # The latest version returns an 302 and contains a location header that is followed
+    # and then downloaded the same way the specific version is downloaded.
+    #
+    # Specific version:
+    #
+    #     https://registry.terraform.io/v1/modules/terraform-aws-modules/sqs/aws/download
+    #
+    # The specific version returns an 204 and then we grab the download url info form the x-terraform-get header.
+    def to_github
+      base_site = "https://registry.terraform.io"
+      base_url = "#{base_site}/v1/modules"
+
+      version = @mod.version.sub(/^v/,'') if @mod.version # v1.0 => 1.0
+      api_url = [base_url, @mod.source, version, "download"].compact.join('/')
+      resp = http_request(api_url)
+
+      case resp.code.to_i
+      when 204
+        download_url = resp.header["x-terraform-get"]
+      when 302
+        next_url = "#{base_site}#{resp.header["location"]}"
+        resp = http_request(next_url)
+        download_url = resp.header["x-terraform-get"]
+      else
+        raise "Unable to lookup up module in Terraform Registry: #{resp}"
+      end
+
+      download_url.sub(%r{/archive/.*},'')
+    end
+
+  private
+    def http_request(url)
+      uri = URI(url)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = uri.scheme == "https"
+      # Total time will be 40s = 20 x 2
+      http.max_retries = 1 # Default is already 1, just  being explicit
+      http.read_timeout = 20 # Sites that dont return in 20 seconds are considered down
+      request = Net::HTTP::Get.new(uri)
+      begin
+         http.request(request) # response
+      rescue Net::OpenTimeout => e # internal ELB but VPC is not configured for Lambda function
+        http_request_error_message(e)
+        puts "The Lambda Function does not seem to have network access to the url. It might not be configured with a VpcConfig.  Please double check that the related vpc variables are configured: @subnet_ids, @vpc_id, @security_group_ingress"
+      rescue Exception => e
+        # Net::ReadTimeout - too slow
+        # Errno::ECONNREFUSED - completely down
+        # SocketError - improper url "dsfjsl" instead of example.com
+        http_request_error_message(e)
+      end
+    end
+  end
+end

data/lib/terraspace_bundler/mod/sync.rb

@@ -0,0 +1,53 @@
+class TerraspaceBundler::Mod
+  class Sync
+    extend Memoist
+    include TB::Helper::Git
+    include TB::Logging
+    include TB::Mod::TmpPaths
+
+    attr_reader :sha
+    def initialize(mod, url)
+      @mod, @url = mod, url
+    end
+
+    def run
+      setup_tmp
+      org_path = "#{cache_root}/#{@mod.org}"
+      FileUtils.mkdir_p(org_path)
+      Dir.chdir(org_path) do
+        name = File.basename(@url).sub('.git','')
+        unless File.exist?(name)
+          sh "git clone #{@url}"
+        end
+
+        Dir.chdir(name) do
+          git "pull"
+          git "submodule update --init"
+          stage(name)
+        end
+      end
+    end
+
+    def stage(name)
+      copy_to_stage(name)
+      switch_to_specific_version(name)
+    end
+
+    def switch_to_specific_version(name)
+      stage_path = stage_path("#{@mod.org}/#{name}")
+      logger.debug "Within: #{stage_path}"
+      Dir.chdir(stage_path) do
+        git "checkout #{@mod.checkout_version}" if @mod.checkout_version
+        @sha = git("rev-parse HEAD").strip
+      end
+    end
+
+    def copy_to_stage(name)
+      cache_path = cache_path("#{@mod.org}/#{name}")
+      stage_path = stage_path("#{@mod.org}/#{name}")
+      FileUtils.rm_rf(stage_path)
+      FileUtils.mkdir_p(File.dirname(stage_path))
+      FileUtils.cp_r(cache_path, stage_path)
+    end
+  end
+end

data/lib/terraspace_bundler/mod/tmp_paths.rb

@@ -0,0 +1,28 @@
+class TerraspaceBundler::Mod
+  module TmpPaths
+    def setup_tmp
+      FileUtils.mkdir_p(cache_root)
+      FileUtils.mkdir_p(stage_root)
+    end
+
+    def tmp_root
+      "/tmp/terraspace/bundler"
+    end
+
+    def cache_root
+      "#{tmp_root}/cache"
+    end
+
+    def stage_root
+      "#{tmp_root}/stage"
+    end
+
+    def cache_path(name)
+      "#{cache_root}/#{name}"
+    end
+
+    def stage_path(name)
+      "#{stage_root}/#{name}"
+    end
+  end
+end

data/lib/terraspace_bundler/setup.rb

@@ -0,0 +1,20 @@
+module TerraspaceBundler
+  class Setup
+    def initialize(options={})
+      @options = options
+    end
+
+    def setup!
+      # Run the DSL to possibly override the config defaults
+      meta = Dsl.new.run
+      export_path = meta[:global][:export_path]
+      TB.config.export_path = export_path if export_path
+
+      # cli --terrafile option
+      if @options[:terrafile]
+        TB.config.terrafile = @options[:terrafile]
+        TB.config.lockfile = "#{@options[:terrafile]}.lock"
+      end
+    end
+  end
+end

data/lib/terraspace_bundler/syncer.rb

@@ -0,0 +1,13 @@
+module TerraspaceBundler
+  class Syncer
+    def initialize(mods)
+      @mods = mods
+    end
+
+    def run
+      @mods.each do |mod|
+        mod.sync
+      end
+    end
+  end
+end