terraspace 2.0.1 → 2.1.0

data/lib/terraspace/cloud/upload.rb

@@ -0,0 +1,53 @@
+module Terraspace::Cloud
+  class Upload < Base
+    def create(data) # data: {stream_id:, type: }
+      zip_path = Package.new(@options.merge(data)).build
+      upload = create_upload(data)
+      upload_project(upload, zip_path)
+      upload
+    end
+
+  private
+    def create_upload(params)
+      upload = api.create_upload(params)
+      if errors?(upload)
+        error_message(upload)
+        exit 1 # Consider: raise exception can rescue higher up
+      else
+        upload
+      end
+    end
+
+    def upload_project(upload, path, retries=0)
+      url = upload['data']['attributes']['url']
+      uri = URI.parse(url)
+      object_content = IO.read(path)
+      sleep 2
+      resp = Net::HTTP.start(uri.host) do |http|
+        http.send_request(
+          'PUT',
+          uri.request_uri,
+          object_content,
+          'content-type' => ''
+        )
+      end
+      return if resp.code =~ /^20/
+
+      log_errors(resp.body, :debug)
+      if resp.body.include?('expired')
+        if retries >= 10 # 15m * 10 = 150m = 2.5h
+          log_errors(resp.body, :info)
+          exit 1
+        end
+        logger.debug("Retrying upload")
+        upload = create_upload(upload_id: upload['data']['id'])
+        upload_project(upload, path, retries+1)
+      end
+    end
+
+    def log_errors(body, level=:debug)
+      logger.send(level, "ERROR: Uploader uploading code")
+      logger.send(level, "resp body #{body}")
+    end
+  end
+end

data/lib/terraspace/cloud/vcs/base.rb

@@ -0,0 +1,6 @@
+class Terraspace::Cloud::Vcs
+  class Base
+    extend Memoist
+    include Terraspace::Util::Logging
+  end
+end

data/lib/terraspace/cloud/vcs/ci_env.rb

@@ -0,0 +1,15 @@
+class Terraspace::Cloud::Vcs
+  class CiEnv
+    extend Memoist
+    include Terraspace::Util::Logging
+
+    def vars
+      klass = Terraspace::Cloud::Ci.detect
+      if klass
+        klass.new.vars.compact # remove items with nil values
+      else
+        {}
+      end
+    end
+  end
+end

data/lib/terraspace/cloud/vcs/commenter.rb

@@ -0,0 +1,75 @@
+class Terraspace::Cloud::Vcs
+  module Commenter
+    extend Memoist
+
+    # record is plan or update
+    def pr_comment(record, cost)
+      return unless able_to_comment?
+
+      resp = cloud_comment.get(record, cost)
+      body = resp['data']['attributes']['body'] if resp
+      if body
+        vcs.comment(body)
+      else
+        logger.info "WARN: Unable to post a PR comment"
+      end
+    end
+
+    def cloud_comment
+      Terraspace::Cloud::Comment.new(@options.merge(stack: @mod.name, kind: kind))
+    end
+    memoize :cloud_comment
+
+    def vcs
+      # IE: TerraspaceVcsGithub::Interface.new
+      Terraspace::Cloud::Vcs.detect(full_repo: vcs_vars[:full_repo], pr_number: vcs_vars[:pr_number])
+    end
+    memoize :vcs
+
+    #
+    # full_repo       # env or .git * min required for pr url
+    # host            # env or .git * min required for pr url
+    # branch_name     # env or .git
+    # sha             # env or .git
+    # dirty           # env or .git
+    # pr_number       # env         * min required for pr url
+    #
+    # commit_message  # env
+    # build_system    # env
+    # build_id        # env
+    # build_number    # env
+    # build_type      # env
+    #
+    # build_url
+    # commit_url
+    # branch_url
+    # pr_url
+    #
+    def vcs_vars
+      ci_env = CiEnv.new
+      local_git = LocalGit.new
+      local_env = LocalEnv.new
+      vars = local_git.vars.merge(ci_env.vars).merge(local_env.vars)
+      finalize_vars(local_git, vars)
+    end
+    memoize :vcs_vars
+
+    # finalize branch_url since info is set from multiple layers
+    def finalize_vars(local_git, vars)
+      if local_git.vcs_class # IE: LocalGit::Github
+        vars[:branch_url] ||= local_git.vcs_class.new(vars).branch_url # IE: LocalGit::Github#branch_url
+      end
+      vars
+    end
+
+    def able_to_comment?
+      !!(vcs && vcs_vars[:full_repo] && vcs_vars[:pr_number])
+    end
+
+    # stored on cloud: kind can be apply or destroy. its extra info
+    def kind
+      is_destroy = Terraspace.command?("down") || @options[:args]&.include?('--destroy') || @options[:destroy]
+      is_destroy ? "destroy" : "apply"
+    end
+  end
+end

data/lib/terraspace/cloud/vcs/interface.rb

@@ -0,0 +1,14 @@
+class Terraspace::Cloud::Vcs
+  module Interface
+    extend Memoist
+    include Terraspace::Util::Logging
+
+    MARKER = "<!-- terraspace marker -->"
+
+    attr_reader :full_repo, :pr_number
+    def initialize(options={})
+      @full_repo = options[:full_repo]
+      @pr_number = options[:pr_number]
+    end
+  end
+end

data/lib/terraspace/cloud/vcs/local_env.rb

@@ -0,0 +1,25 @@
+class Terraspace::Cloud::Vcs
+  class LocalEnv < Base
+    def vars
+      # Do not set any default values. These take highest precedence and will override LocalGit and CiVars
+      {
+        host: ENV['TS_VCS_HOST'] || ENV['VCS_HOST'] || ENV['GIT_HOST'],
+        full_repo: ENV['TS_VCS_REPO'] || ENV['VCS_REPO'] || ENV['GIT_REPO'],
+        branch_name: ENV['TS_VCS_BRANCH'],
+        # urls
+        commit_url: ENV['TS_VCS_COMMIT_URL'],
+        branch_url: ENV['TS_VCS_BRANCH_URL'],
+        pr_url: ENV['TS_VCS_PR_URL'],
+        build_url: ENV['TS_VCS_BUILD_URL'],
+        # additional properties
+        build_type: ENV['TS_VCS_BUILD_TYPE'],
+        # pr_number: ENV['TS_VCS_PR_NUMBER'] || ENV['PR_NUMBER'] || ENV['MR_NUMBER'], # handle in LocalGit::Base so can compute pr_url
+        sha: ENV['TS_VCS_SHA'],
+        # additional properties
+        commit_message: ENV['TS_VCS_COMMIT_MESSAGE'],
+        build_id: ENV['TS_VCS_BUILD_ID'],
+        build_number: ENV['TS_VCS_BUILD_NUMBER'],
+      }.compact # remove items with nil values
+    end
+  end
+end

data/lib/terraspace/cloud/{ci/vcs → vcs/local_git}/base.rb

@@ -1,4 +1,4 @@
-class Terraspace::Cloud::Ci::Vcs
+class Terraspace::Cloud::Vcs::LocalGit
   class Base
     extend Memoist
 
@@ -10,11 +10,14 @@ class Terraspace::Cloud::Ci::Vcs
       {
         commit_url: commit_url,  # implemented by subclass
         branch_url: branch_url,  # implemented by subclass
+        # pr_url handled when PR_NUMBER set by user outside of ci env. ci plugin pr_url takes higher precedence though
+        pr_number: pr_number,
+        pr_url: pr_url,
       }
     end
 
-    def merged_vars
-      @vars.merge(vars)
+    def pr_number
+      ENV['TS_VCS_PR_NUMBER'] || ENV['PR_NUMBER'] || ENV['MR_NUMBER']
     end
 
     class << self

data/lib/terraspace/cloud/vcs/local_git/bitbucket.rb

@@ -0,0 +1,17 @@
+class Terraspace::Cloud::Vcs::LocalGit
+  class Bitbucket < Base
+    def commit_url
+      "#{host}/#{full_repo}/commits/#{sha}" if sha
+    end
+
+    def branch_url
+      "#{host}/#{full_repo}/branch/#{branch_name}" if branch_name
+    end
+
+    # Also computed in ci plugins which detects it from the ci env.
+    # Also handling here for case when user provides PR_NUMBER and GIT_REPO outside of normal CI env.
+    def pr_url
+      "#{host}/#{full_repo}/pull-requests/#{pr_number}" if pr_number
+    end
+  end
+end

data/lib/terraspace/cloud/vcs/local_git/github.rb

@@ -0,0 +1,17 @@
+class Terraspace::Cloud::Vcs::LocalGit
+  class Github < Base
+    def commit_url
+      "#{host}/#{full_repo}/commits/#{sha}" if sha
+    end
+
+    def branch_url
+      "#{host}/#{full_repo}/tree/#{branch_name}" if branch_name
+    end
+
+    # Also computed in ci plugins which detects it from the ci env.
+    # Also handling here for case when user provides PR_NUMBER and GIT_REPO outside of normal CI env.
+    def pr_url
+      "#{host}/#{full_repo}/pull/#{pr_number}" if pr_number
+    end
+  end
+end

data/lib/terraspace/cloud/vcs/local_git/gitlab.rb

@@ -0,0 +1,17 @@
+class Terraspace::Cloud::Vcs::LocalGit
+  class Gitlab < Base
+    def commit_url
+      "#{host}/#{full_repo}/-/commits/#{sha}" if sha
+    end
+
+    def branch_url
+      "#{host}/#{full_repo}/-/tree/#{branch_name}" if branch_name
+    end
+
+    # Also computed in ci plugins which detects it from the ci env.
+    # Also handling here for case when user provides PR_NUMBER and GIT_REPO outside of normal CI env.
+    def pr_url
+      "#{host}/#{full_repo}/-/merge_requests/#{pr_number}" if pr_number
+    end
+  end
+end

data/lib/terraspace/cloud/{ci/manual.rb → vcs/local_git.rb}

@@ -1,17 +1,23 @@
-class Terraspace::Cloud::Ci
-  class Manual
-    extend Memoist
-    include Terraspace::Util::Logging
-
+class Terraspace::Cloud::Vcs
+  class LocalGit < Base
     def vars
       if git_repo? && git_installed?
-        vars_data
+        provider_vars = vcs_class ? vcs_class.new(base_vars).vars : {}
+        base_vars.merge(provider_vars).compact # remove items with nil values
       else
         { build_system: "manual" }
       end
     end
 
-    def vars_data
+    def vcs_class
+      case host
+      when /github/ then Github
+      when /gitlab/ then Gitlab
+      when /bitbucket/ then Bitbucket
+      end
+    end
+
+    def base_vars
       {
         build_system: "manual",   # required
         host: host,
@@ -19,8 +25,8 @@ class Terraspace::Cloud::Ci
         branch_name: branch_name,
         sha: sha,
         dirty: dirty?,
-        # commit_url: commit_url,  # provided by core
-        # branch_url: branch_url,  # provided by core
+        # commit_url: commit_url,  # provided by provider vars
+        # branch_url: branch_url,  # provided by provider vars
       }
     end
 
@@ -46,7 +52,9 @@ class Terraspace::Cloud::Ci
     #   git@bitbucket.org: => https://bitbucket.org/
     #   git@gitlab.com:    => https://gitlab.com/
     def git_url
-      out = git "config --get remote.origin.url"
+      remotes = git("remote").strip.split("\n")
+      remote_name = remotes.size == 1 ? remotes[0] : "origin"
+      out = git "config --get remote.#{remote_name}.url"
       out.sub(/\.git/,'').sub(/^git@/,'https://').sub(/\.(.*):/,'.\1/')
     end
 

data/lib/terraspace/cloud/vcs.rb

@@ -0,0 +1,21 @@
+module Terraspace::Cloud
+  class Vcs
+    class << self
+      @@name = nil
+      def register(options={})
+        @@name = options[:name]
+      end
+
+      # IE: TerraspaceVcsGithub::Interface.new
+      def detect(options)
+        name = detect_name
+        "TerraspaceVcs#{name.camelize}::Interface".constantize.new(options) if name
+      end
+
+      def detect_name
+        # allow user to override vcs.name
+        Terraspace.config.cloud.vcs.name || @@name
+      end
+    end
+  end
+end

data/lib/terraspace/compiler/strategy/tfvar/layer.rb

@@ -41,7 +41,7 @@ class Terraspace::Compiler::Strategy::Tfvar
     def paths
       project_paths = full_paths(project_tfvars_dir)
       stack_paths   = full_paths(stack_tfvars_dir)
-      paths = project_paths + stack_paths
+      paths = (project_paths + stack_paths).uniq
       show_layers(paths)
       paths.select do |path|
         File.exist?(path)

data/lib/terraspace/core.rb

@@ -18,6 +18,14 @@ module Terraspace
       ENV['TS_EXTRA'] unless ENV['TS_EXTRA'].blank?
     end
 
+    def project
+      if ENV['TS_PROJECT'].blank?
+        config.cloud.project
+      else
+        ENV['TS_PROJECT']
+      end
+    end
+
     @@root = nil
     def root
       @@root ||= ENV['TS_ROOT'] || Dir.pwd

data/lib/terraspace/logger.rb

@@ -14,7 +14,7 @@ module Terraspace
       else
         super # use the configured formatter
       end
-      out = line =~ /\n$/ ? line : "#{line}\n"
+      out = line.force_encoding('UTF-8') =~ /\n$/ ? line : "#{line}\n"
       @@buffer << out
       out
     end
@@ -45,7 +45,8 @@ module Terraspace
       end
 
       def logs
-        @@buffer.join('')
+        # force_encoding https://jch.github.io/posts/2013-03-05-ruby-incompatible-encoding.html
+        @@buffer.map { |s| s.force_encoding('UTF-8') }.join('')
       end
 
       # for test framework

data/lib/terraspace/plugin/expander/interface.rb

@@ -96,10 +96,11 @@ module Terraspace::Plugin::Expander
       @mod.name
     end
 
-    def app;   Terraspace.app   ; end
-    def role;  Terraspace.role  ; end
-    def env;   Terraspace.env   ; end
-    def extra; Terraspace.extra ; end
+    def app;     Terraspace.app     ; end
+    def env;     Terraspace.env     ; end
+    def extra;   Terraspace.extra   ; end
+    def project; Terraspace.project ; end
+    def role;    Terraspace.role    ; end
 
     def type_instance
       [type, instance].reject { |s| s.blank? }.join('-')
@@ -114,10 +115,6 @@ module Terraspace::Plugin::Expander
       Terraspace.cache_root
     end
 
-    def project
-      Terraspace.config.cloud.project
-    end
-
     # So default config works:
     #    config.cache_dir = ":REGION/:ENV/:BUILD_DIR"
     # For when folks configure it with the http backend for non-cloud providers

data/lib/terraspace/shell/error.rb

@@ -35,7 +35,7 @@ class Terraspace::Shell
 
     def message
       # For error messages, terraform lines from buffer do not contain newlines. So join with newline
-      @lines.join("\n")
+      @lines.map { |l| l.force_encoding('UTF-8') }.join("\n")
     end
 
     def shared_cache_error?

data/lib/terraspace/terraform/runner.rb

@@ -4,7 +4,7 @@ module Terraspace::Terraform
     include Terraspace::Hooks::Concern
     include Terraspace::Util
 
-    attr_reader :name
+    attr_reader :name, :shell_exception
     def initialize(name, options={})
       @name = name
       super(options)
@@ -14,12 +14,7 @@ module Terraspace::Terraform
       time_took do
         terraform(name, args)
       end
-
-      run_internal_hook(:after, name) # always run so plan and apply get saved to tsc
-      if @shell_error
-        logger.error @shell_error.message.color(:red)
-        exit 1 if name == "plan" && Terraspace.command?("up")
-      end
+      @success
     end
 
     # default at end in case of redirection. IE: terraform output > /path
@@ -51,15 +46,14 @@ module Terraspace::Terraform
 
       params = args.flatten.join(' ')
       command = "terraform #{name} #{params}".squish
-      @shell_error = nil
+      @shell_exception = nil
       run_hooks("terraform.rb", name) do
         Backend.new(@mod).create
-        run_internal_hook(:before, name)
         begin
           Terraspace::Shell.new(@mod, command, @options.merge(env: custom.env_vars)).run
           @success = true
         rescue Terraspace::ShellError => exception
-          @shell_error = exception
+          @shell_exception = exception
           @success = false
         end
       end
@@ -73,18 +67,6 @@ module Terraspace::Terraform
       end
     end
 
-    def run_internal_hook(type, name)
-      begin
-        class_name = "Terraspace::Terraform::Ihooks::#{type.to_s.classify}::#{name.classify}"
-        klass = class_name.constantize
-      rescue NameError => e
-        logger.debug "DEBUG: #{e.class} #{e.message}".color(:red)
-        return
-      end
-      ihook = klass.new(name, @options.merge(success: @success))
-      ihook.run
-    end
-
     @@current_dir_message_shown = false
     def current_dir_message
       return if @@current_dir_message_shown

data/lib/terraspace/util/popen.rb

@@ -0,0 +1,67 @@
+require 'open3'
+
+module Terraspace::Util
+  module Popen
+    include Logging
+
+    # Similar to Terraspace::Shell#popen3
+    # More generalized. Useful for plugins
+    def popen(command, options={})
+      Open3.popen3(command) do |stdin, stdout, stderr, wait_thread|
+        handle_streams(stdin, stdout, stderr, options)
+        status = wait_thread.value.exitstatus
+        unless status == 0
+          logger.error "Error running command #{command}".color(:red)
+          exit 1
+        end
+      end
+    end
+
+    BLOCK_SIZE = Integer(ENV['TS_BUFFER_BLOCK_SIZE'] || 102400)
+    def handle_streams(stdin, stdout, stderr, options)
+      files = [stdout, stderr]
+      until all_eof?(files) do
+        ready = IO.select(files)
+        next unless ready
+
+        readable = ready[0]
+        readable.each do |f|
+          buffer = f.read_nonblock(BLOCK_SIZE, exception: false)
+          next unless buffer
+
+          lines = buffer.split("\n")
+          lines.each do |line|
+            if f.fileno == stdout.fileno
+              handle_stdout(line, options)
+            else
+              handle_stderr(line, options)
+            end
+          end
+        end
+      end
+    end
+
+    def handle_stdout(line, options={})
+      newline = options[:newline].nil? ? true : options[:newline]
+      filter = options[:filter]
+
+      # Terraspace logger has special stdout method so original terraform output
+      # can be piped to jq. IE:
+      #   terraspace show demo --json | jq
+      if logger.respond_to?(:stdout) && !options[:log_to_stderr]
+        logger.stdout(line, newline: newline) unless line.include?(filter)
+      else
+        logger.info(line) unless line.include?(filter)
+      end
+    end
+
+    def handle_stderr(line, options={})
+      logger.error(line) unless line.include?(options[:filter])
+    end
+
+    def all_eof?(files)
+      files.find { |f| !f.eof }.nil?
+    end
+
+  end
+end

data/lib/terraspace/version.rb

@@ -1,3 +1,3 @@
 module Terraspace
-  VERSION = "2.0.1"
+  VERSION = "2.1.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: terraspace
 version: !ruby/object:Gem::Version
-  version: 2.0.1
+  version: 2.1.0
 platform: ruby
 authors:
 - Tung Nguyen
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-06-14 00:00:00.000000000 Z
+date: 2022-07-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -382,12 +382,14 @@ files:
 - ".cody/none/iam_role.rb"
 - ".cody/none/project.rb"
 - ".cody/shared/script/install.sh"
+- ".cody/shared/script/install/infracost.sh"
 - ".cody/shared/script/install/terraform.sh"
 - ".cody/shared/script/install/terraspace.sh"
 - ".cody/shared/script/update/gemfile.sh"
 - ".cody/unit/bin/build.sh"
 - ".cody/unit/buildspec.yml"
 - ".cody/unit/project.rb"
+- ".github/FUNDING.yml"
 - ".github/ISSUE_TEMPLATE.md"
 - ".github/ISSUE_TEMPLATE/bug_report.md"
 - ".github/ISSUE_TEMPLATE/documentation.md"
@@ -450,7 +452,6 @@ files:
 - lib/templates/plugin/ci/lib/%gem_name%.rb.tt
 - lib/templates/plugin/ci/lib/%gem_name%/autoloader.rb.tt
 - lib/templates/plugin/ci/lib/%gem_name%/interface.rb.tt
-- lib/templates/plugin/ci/lib/%gem_name%/pr.rb.tt
 - lib/templates/plugin/ci/lib/%gem_name%/vars.rb.tt
 - lib/templates/plugin/ci/lib/%gem_name%/version.rb.tt
 - lib/templates/plugin/ci/spec/%gem_name%_spec.rb.tt
@@ -645,26 +646,34 @@ files:
 - lib/terraspace/cloud/api/cani.rb
 - lib/terraspace/cloud/api/concern.rb
 - lib/terraspace/cloud/api/concern/errors.rb
-- lib/terraspace/cloud/api/concern/record.rb
 - lib/terraspace/cloud/api/http_methods.rb
 - lib/terraspace/cloud/api/validate.rb
 - lib/terraspace/cloud/base.rb
 - lib/terraspace/cloud/ci.rb
-- lib/terraspace/cloud/ci/generic.rb
-- lib/terraspace/cloud/ci/manual.rb
-- lib/terraspace/cloud/ci/vcs.rb
-- lib/terraspace/cloud/ci/vcs/base.rb
-- lib/terraspace/cloud/ci/vcs/bitbucket.rb
-- lib/terraspace/cloud/ci/vcs/github.rb
-- lib/terraspace/cloud/ci/vcs/gitlab.rb
+- lib/terraspace/cloud/comment.rb
 - lib/terraspace/cloud/context.rb
-- lib/terraspace/cloud/folder.rb
-- lib/terraspace/cloud/folder/base.rb
-- lib/terraspace/cloud/folder/package.rb
-- lib/terraspace/cloud/folder/tidy.rb
-- lib/terraspace/cloud/folder/uploader.rb
+- lib/terraspace/cloud/cost.rb
+- lib/terraspace/cloud/cost/infracost.rb
+- lib/terraspace/cloud/git.rb
 - lib/terraspace/cloud/plan.rb
+- lib/terraspace/cloud/stream.rb
+- lib/terraspace/cloud/streamer.rb
 - lib/terraspace/cloud/update.rb
+- lib/terraspace/cloud/upload.rb
+- lib/terraspace/cloud/upload/base.rb
+- lib/terraspace/cloud/upload/package.rb
+- lib/terraspace/cloud/upload/tidy.rb
+- lib/terraspace/cloud/vcs.rb
+- lib/terraspace/cloud/vcs/base.rb
+- lib/terraspace/cloud/vcs/ci_env.rb
+- lib/terraspace/cloud/vcs/commenter.rb
+- lib/terraspace/cloud/vcs/interface.rb
+- lib/terraspace/cloud/vcs/local_env.rb
+- lib/terraspace/cloud/vcs/local_git.rb
+- lib/terraspace/cloud/vcs/local_git/base.rb
+- lib/terraspace/cloud/vcs/local_git/bitbucket.rb
+- lib/terraspace/cloud/vcs/local_git/github.rb
+- lib/terraspace/cloud/vcs/local_git/gitlab.rb
 - lib/terraspace/command.rb
 - lib/terraspace/compiler/basename.rb
 - lib/terraspace/compiler/cleaner.rb
@@ -773,13 +782,6 @@ files:
 - lib/terraspace/terraform/args/pass.rb
 - lib/terraspace/terraform/args/shorthands.rb
 - lib/terraspace/terraform/args/thor.rb
-- lib/terraspace/terraform/ihooks/after/apply.rb
-- lib/terraspace/terraform/ihooks/after/destroy.rb
-- lib/terraspace/terraform/ihooks/after/plan.rb
-- lib/terraspace/terraform/ihooks/base.rb
-- lib/terraspace/terraform/ihooks/before/apply.rb
-- lib/terraspace/terraform/ihooks/before/destroy.rb
-- lib/terraspace/terraform/ihooks/before/plan.rb
 - lib/terraspace/terraform/remote_state/fetcher.rb
 - lib/terraspace/terraform/remote_state/marker/output.rb
 - lib/terraspace/terraform/remote_state/marker/pretty_tracer.rb
@@ -802,6 +804,7 @@ files:
 - lib/terraspace/tester/meta.rb
 - lib/terraspace/util.rb
 - lib/terraspace/util/logging.rb
+- lib/terraspace/util/popen.rb
 - lib/terraspace/util/pretty.rb
 - lib/terraspace/util/sure.rb
 - lib/terraspace/version.rb