terraspace 2.0.3 → 2.1.2

data/lib/terraspace/cloud/api.rb

@@ -16,18 +16,42 @@ module Terraspace::Cloud
       "orgs/#{@org}/projects/#{@project}/stacks/#{@stack}"
     end
 
-    def create_upload
-      post("#{stack_path}/uploads", @options)
+    # data: {stream_id:}
+    def create_upload(data)
+      post("#{stack_path}/uploads", @options.merge(data))
     end
 
-    # record_attrs: {upload_id: "upload-nRPSpyWd65Ps6978", kind: "apply", stack_id: '...'}
+    # data: {stream_id:}
+    def create_stream(data)
+      post("#{stack_path}/streams", @options.merge(data))
+    end
+
+    # data: {id:, success:}
+    def complete_stream(data={})
+      post("#{stack_path}/streams/#{data[:id]}/complete", @options.merge(data))
+    end
+
+    # data: {upload_id: "upload-nRPSpyWd65Ps6978", kind: "apply", stack_id: '...'}
     def create_plan(data)
-      post("#{stack_path}/plans", data.merge(@options))
+      post("#{stack_path}/plans", @options.merge(data))
     end
 
     # data: {upload_id: "upload-nRPSpyWd65Ps6978", kind: "apply", stack_id: '...'}
     def create_update(data)
-      post("#{stack_path}/updates", data.merge(@options))
+      post("#{stack_path}/updates", @options.merge(data))
+    end
+
+    # data: {upload_id: "upload-nRPSpyWd65Ps6978", stack_id: '...'}
+    def create_cost(data)
+      post("#{stack_path}/costs", @options.merge(data))
+    end
+
+    def get_previous_cost(data)
+      get("#{stack_path}/costs/previous", @options.merge(data))
+    end
+
+    def get_comment(data)
+      get("#{stack_path}/comment", @options.merge(data))
     end
   end
 end

data/lib/terraspace/cloud/base.rb

@@ -9,45 +9,39 @@ module Terraspace::Cloud
       super
       @cani = options[:cani]
       @kind = options[:kind]
-      @success = options[:success]
+      @vcs_vars = options[:vcs_vars]
       setup_context(options)
     end
 
-    def stage_attrs
-      status = @success ? "success" : "fail"
+    def stage_attrs(success)
+      status = success_status(success)
       attrs = {
         status: status,
         kind: @kind,
         terraspace_version: check.terraspace_version,
         terraform_version: check.terraform_version,
       }
-      attrs.merge!(ci_vars) if ci_vars
+      attrs.merge!(@vcs_vars)
       attrs
     end
 
-    def check
-      Terraspace::CLI::Setup::Check.new
-    end
-    memoize :check
-
-    def pr_comment(url)
-      ci.comment(url) if ci.respond_to?(:comment)
+    def cloud_upload
+      Upload.new(@options)
     end
+    memoize :cloud_upload
 
-    def ci_vars
-      return unless ci
-      if ci.vars[:host]
-        vcs = Ci::Vcs.new(ci.vars)
-        vcs.merged_vars
-      else
-        ci.vars
+    def success_status(success)
+      case success
+      when true then "success"
+      when false then "fail"
+      when nil then "started"
       end
     end
 
-    def ci
-      Terraspace::Cloud::Ci.detect
+    def check
+      Terraspace::CLI::Setup::Check.new
     end
-    memoize :ci
+    memoize :check
 
     def sh(command, exit_on_fail: true)
       logger.debug "=> #{command}"
@@ -86,12 +80,5 @@ module Terraspace::Cloud
         line.include?(' cancelled')
       end
     end
-
-    def terraspace_cloud_info(result)
-      data = result['data']
-      url = data['attributes']['url']
-      logger.info "Terraspace Cloud #{url}"
-      url
-    end
   end
 end

data/lib/terraspace/cloud/ci.rb

@@ -18,8 +18,6 @@ module Terraspace::Cloud
       end
 
       def detect
-        return Generic if ENV['TS_CI_REPO']
-
         detected = meta.find do |data|
           env_key = data[:env_key] # IE: ENV['GITHUB_ACTIONS']
           env_value = data[:env_value] # IE: "string" or /pattern/
@@ -30,12 +28,7 @@ module Terraspace::Cloud
             ENV[env_key] # only env_key
           end
         end
-        klass = if detected
-          interface_class(detected)
-        else
-          Manual
-        end
-        klass.new
+        interface_class(detected) if detected
       end
 
       # IE: TerraspaceCiGithub::Interface

data/lib/terraspace/cloud/comment.rb

@@ -0,0 +1,28 @@
+module Terraspace::Cloud
+  class Comment < Base
+    def get(record, cost)
+      return unless Terraspace.cloud?
+
+      params = {}
+      params[:record_id] = record['data']['id']
+      params[:cost_id] = cost['data']['id'] if cost
+
+      sleep 1 # delay a second since job is queued
+      resp = nil
+      Timeout::timeout(20) do
+        loop do
+          resp = api.get_comment(params)
+          break if resp['data']['attributes']['status'] == 'completed'
+          sleep 2
+        end
+      end
+      resp
+    rescue Timeout::Error
+      # nil
+    end
+
+    def cani?(exit_on_error: true)
+      api.create_cost(cani: 1, exit_on_error: exit_on_error)
+    end
+  end
+end

data/lib/terraspace/cloud/context.rb

@@ -3,6 +3,7 @@ module Terraspace::Cloud
     include Terraspace::Cloud::Api::Validate
 
     def setup_context(options)
+      return unless Terraspace.cloud? # else validate of @org errors
       cloud = Terraspace.config.cloud
       @org = cloud.org
       @project = cloud.project

data/lib/terraspace/cloud/cost/infracost.rb

@@ -0,0 +1,80 @@
+class Terraspace::Cloud::Cost
+  class Infracost
+    extend Memoist
+    include Terraspace::Util::Popen
+
+    def initialize(options={})
+      @cloud_stack_name = options[:cloud_stack_name]
+    end
+
+    def name
+      "infracost"
+    end
+
+    def run(output_dir=".terraspace-cache/.cache2/cost")
+      unless infracost_installed?
+        logger.info "WARN: infracost not installed. Please install infracost first: https://www.infracost.io/docs/".color(:yellow)
+        logger.info "Not running cost estimate"
+        return false
+      end
+      unless api_key?
+        logger.info "WARN: infracost api key not configured. Please set the environment variable INFRACOST_API_KEY".color(:yellow)
+        logger.info "Not running cost estimate"
+        return false
+      end
+
+      previous_cost_available = File.exist?("#{output_dir}/cost_prev.json")
+      commands = [
+        "infracost breakdown --path . --format json --out-file #{output_dir}/cost.json --project-name #{@cloud_stack_name}",
+      ]
+      if previous_cost_available
+        commands << "infracost diff --path #{output_dir}/cost.json --compare-to=#{output_dir}/cost_prev.json --format json --out-file #{output_dir}/cost_diff.json"
+      end
+
+      cost_json_file = previous_cost_available ? "cost_diff.json" : "cost.json"
+      commands += [
+        "infracost output --path #{output_dir}/#{cost_json_file} --format html  --out-file #{output_dir}/cost.html",
+        "infracost output --path #{output_dir}/#{cost_json_file} --format table --out-file #{output_dir}/cost.text",
+      ]
+
+      if comment_format
+        path = previous_cost_available ? "cost_diff.json" : "cost.json"
+        commands << "infracost output --path #{output_dir}/#{path} --format #{comment_format} --out-file #{output_dir}/cost.comment"
+      end
+
+      commands.each do |command|
+        logger.debug "=> #{command}"
+
+        popen(command, filter: "Output saved to ")
+        if command.include?(".text")
+          logger.info IO.read("#{output_dir}/cost.text")
+          logger.info "\n"
+        end
+      end
+    end
+
+    def comment_format
+      name = Terraspace::Cloud::Vcs.detect_name # IE: github
+      format = "#{name}-comment"
+      # These are valid infracost comment format. Note: Not all have terraspace_vcs_* plugin support yet
+      valid = %w[github-comment gitlab-comment azure-repos-comment bitbucket-comment]
+      format if valid.include?(format)
+    end
+
+    def version
+      out = `infracost --version`.strip # Infracost v0.10.6
+      md = out.match(/ v(.*)/)
+      md ? md[1] : out
+    end
+    memoize :version
+
+    def infracost_installed?
+      system "type infracost > /dev/null 2>&1"
+    end
+
+    def api_key?
+      return true if ENV['INFRACOST_API_KEY']
+      File.exist?("#{ENV['HOME']}/.config/infracost/credentials.yml")
+    end
+  end
+end

data/lib/terraspace/cloud/cost.rb

@@ -0,0 +1,68 @@
+module Terraspace::Cloud
+  class Cost < Base
+    # uid of plan or update
+    def create(uid:, stream:)
+      return unless Terraspace.cloud?
+      return unless Terraspace.config.cloud.cost.enabled
+
+      cani?
+      success = build(stream)
+      return unless success # in case infracost not installed
+
+      upload = cloud_upload.create(type: "cost", stream_id: stream['data']['id'])
+      api.create_cost(
+        upload_id: upload['data']['id'],
+        stack_uid: upload['data']['attributes']['stack_id'], # use stack_uid since stack_id is friendly url name
+        uid: uid,
+        cost: cost_attrs,
+      )
+    rescue Terraspace::NetworkError => e
+      logger.warn "WARN: #{e.class} #{e.message}"
+      logger.warn "WARN: Unable to save data to Terraspace cloud"
+    end
+
+    # different from stage_attrs
+    def cost_attrs
+      {
+        provider: provider.name, # IE: infracost
+        provider_version: provider.version,
+        terraspace_version: check.terraspace_version,
+        terraform_version: check.terraform_version,
+      }
+    end
+
+    def build(stream)
+      success = nil
+      clean_cache2_stage
+      download_previous_cost(stream)
+      # .terraspace-cache/dev/stacks/demo
+      Dir.chdir(@mod.cache_dir) do
+        logger.info "Running cost estimate..."
+        success = provider.run
+      end
+      success
+    end
+
+    def download_previous_cost(stream)
+      stack_id = stream['data']['attributes']['stack_id'] # stack-SHWx8fW5FbDKg3QK
+      cost = api.get_previous_cost(stack_uid: stack_id)
+      download_url = cost['data']['attributes']['download_url'] # could be nil
+      return unless download_url
+      uri = URI(download_url)
+      json = Net::HTTP.get(uri) # => JSON String
+      dest = "#{@mod.cache_dir}/.terraspace-cache/.cache2/cost/cost_prev.json"
+      FileUtils.mkdir_p(File.dirname(dest))
+      IO.write(dest, json)
+    end
+
+    def provider
+      Terraspace::Cloud::Cost::Infracost.new(cloud_stack_name: cloud_stack_name) # only provider currently supported
+    end
+    memoize :provider
+
+    def cani?(exit_on_error: true)
+      return true unless Terraspace.cloud?
+      api.create_cost(cani: 1, exit_on_error: exit_on_error)
+    end
+  end
+end

data/lib/terraspace/cloud/plan.rb

@@ -2,29 +2,42 @@ module Terraspace::Cloud
   class Plan < Base
     include Terraspace::CLI::Concerns::PlanPath
 
-    def run
-      return unless record?
+    def setup
+      return unless Terraspace.cloud?
+      cani?
+
+      return unless @mod.out_option
+      return if @mod.out_option =~ %r{^/} # not need to create parent dir for copy with absolute path
 
-      build
-      folder = Folder.new(@options.merge(type: "plan"))
-      upload = folder.upload_data # returns upload record
-      result = api.create_plan(
-        upload_id: upload['id'],
-        stack_uid: upload['stack_id'], # use stack_uid since stack_id is friendly url name
-        plan: stage_attrs,
+      name = @mod.out_option.sub("#{Terraspace.root}/",'')
+      dest = "#{@mod.cache_dir}/#{name}"
+      FileUtils.mkdir_p(File.dirname(dest))
+    end
+
+    def create(success, stream)
+      return unless Terraspace.cloud?
+      return unless record?
+      build(success)
+      upload = cloud_upload.create(type: "plan", stream_id: stream['data']['id'])
+      api.create_plan(
+        upload_id: upload['data']['id'],
+        stack_uid: upload['data']['attributes']['stack_id'], # use stack_uid since stack_id is friendly url name
+        plan: stage_attrs(success),
       )
-      url = terraspace_cloud_info(result)
-      pr_comment(url)
     rescue Terraspace::NetworkError => e
       logger.warn "WARN: #{e.class} #{e.message}"
       logger.warn "WARN: Unable to save data to Terraspace cloud"
     end
 
-    def cani?
-      api.create_plan(cani: 1)
+    def create_stream
+      return unless Terraspace.cloud?
+      api.create_stream("plan")
+    rescue Terraspace::NetworkError => e
+      logger.warn "WARN: #{e.class} #{e.message}"
+      logger.warn "WARN: Unable to save data to Terraspace cloud"
     end
 
-    def build
+    def build(success)
       clean_cache2_stage
       # .terraspace-cache/dev/stacks/demo
       Dir.chdir(@mod.cache_dir) do
@@ -32,7 +45,7 @@ module Terraspace::Cloud
 
         IO.write("#{plan_dir}/plan.log", Terraspace::Logger.logs)
 
-        return unless @success
+        return unless success
         return if File.empty?(plan_path)
 
         out_option_root_path = "#{Terraspace.root}/#{plan_path}"
@@ -43,5 +56,10 @@ module Terraspace::Cloud
         sh "terraform show -json #{plan_path} > #{json}"
       end
     end
+
+    def cani?(exit_on_error: true)
+      return true unless Terraspace.cloud?
+      api.create_plan(cani: 1, exit_on_error: exit_on_error)
+    end
   end
 end

data/lib/terraspace/cloud/stream.rb

@@ -0,0 +1,113 @@
+module Terraspace::Cloud
+  class Stream < Base
+    def open(command)
+      return unless Terraspace.cloud?
+      @command = command # down, up, plan
+      @stream = create_stream
+      logger.info "Live Stream: #{@stream['data']['attributes']['url']}"
+      @stream_thread ||= Thread.new do
+        loop do
+          upload_stream
+          create_cloud_record
+          break if @end_streaming
+          sleep 1
+        end
+      end
+      @stream
+    end
+
+    def create_cloud_record
+      return if @created
+      # Dont create records unless changes are detected or user configure to always create recrods
+      return unless record_all? || yes? || changes?
+
+      case @command
+      when "plan"
+        create_plan
+      when "up", "down"
+        create_update
+      end
+      @created = true
+    end
+
+    def record_all?
+      Terraspace.config.cloud.record == "all"
+    end
+
+    # Notes: https://gist.github.com/tongueroo/12ca3bec3043fce5e9b52b8580da6b6c
+    def changes?
+      buffer = Terraspace::Logger.buffer.map { |s| s.force_encoding('UTF-8') }
+      will_found = buffer.detect { |s| s.include?("Terraform will perform") }
+      saved_found = buffer.detect { |s| s.include?("Saved the plan to:") }
+      !!(will_found && saved_found)
+    end
+
+    def yes?
+      Terraspace::Logger.stdin_capture == 'yes'
+    end
+
+    def create_plan
+      api.create_plan(
+        stream_id: @stream['data']['id'],
+        plan: stage_attrs(nil),
+      )
+    end
+
+    def create_update
+      api.create_update(
+        stream_id: @stream['data']['id'],
+        update: stage_attrs(nil),
+      )
+    end
+
+    def close(success, exception)
+      return unless @stream
+      @end_streaming = true
+      @stream_thread.join
+      status = success ? "success" : "fail"
+      api.complete_stream(id: @stream['data']['id'], status: status, exception: !!exception)
+    end
+
+    def upload_stream(retries=0)
+      stream_url = @stream['data']['attributes']['upload_url']
+      url = stream_url
+      uri = URI.parse(url)
+      object_content = Terraspace::Logger.logs
+      resp = Net::HTTP.start(uri.host) do |http|
+        http.send_request(
+          'PUT',
+          uri.request_uri,
+          object_content,
+          'content-type' => ''
+        )
+      end
+
+      return if resp.code =~ /^20/
+
+      log_errors(resp.body, :debug)
+      if resp.body.include?('expired')
+        if retries >= 10 # 15m * 10 = 150m = 2.5h TODO: CHANGE
+          log_errors(resp.body, :info)
+          # raise "RetriesExceeded"
+          exit 1
+        end
+        logger.debug("Retrying upload")
+        @stream = create_stream(stream_id: @stream['data']['id'])
+        upload_stream(retries+1)
+      end
+    end
+
+    def create_stream(params={})
+      api.create_stream(params.merge(command: @command))
+    rescue Terraspace::NetworkError => e
+      logger.warn "WARN: Error calling Terraspace cloud"
+      logger.warn "WARN: #{e.class} #{e.message}"
+    end
+
+    def log_errors(body, level=:debug)
+      logger.send(level, "ERROR: Stream uploading logs")
+      logger.send(level, "resp body #{body}")
+    end
+
+  end
+end

data/lib/terraspace/cloud/streamer.rb

@@ -0,0 +1,9 @@
+module Terraspace::Cloud
+  module Streamer
+    extend Memoist
+    def cloud_stream
+      Terraspace::Cloud::Stream.new(@options.merge(stack: @mod.name, kind: kind, vcs_vars: vcs_vars))
+    end
+    memoize :cloud_stream
+  end
+end

data/lib/terraspace/cloud/update.rb

@@ -1,37 +1,37 @@
 module Terraspace::Cloud
   class Update < Base
-    def run
+    def create(success, stream)
+      return unless Terraspace.cloud?
       return unless record?
 
-      build
-      folder = Folder.new(@options.merge(type: @kind))
-      upload = folder.upload_data # returns upload record
-      result = api.create_update(
-        upload_id: upload['id'],
-        stack_uid: upload['stack_id'], # use stack_uid since stack_id is friendly url name
-        update: stage_attrs,
-      )
-      url = terraspace_cloud_info(result)
-      pr_comment(url)
+      build(success)
+      upload = cloud_upload.create(type: "update", stream_id: stream['data']['id'])
+      params = {
+        upload_id: upload['data']['id'],
+        stack_uid: upload['data']['attributes']['stack_id'], # use stack_uid since stack_id is friendly url name
+        update: stage_attrs(success),
+      }
+      api.create_update(params)
     end
 
-    def cani?
-      api.create_update(cani: 1)
-    end
-
-    def build
+    def build(success)
       clean_cache2_stage
       # .terraspace-cache/dev/stacks/demo
       Dir.chdir(@mod.cache_dir) do
-        cache2_path = ".terraspace-cache/.cache2/#{@kind}"
+        cache2_path = ".terraspace-cache/.cache2/update"
         FileUtils.mkdir_p(cache2_path)
 
-        IO.write("#{cache2_path}/#{@kind}.log", Terraspace::Logger.logs)
-        return unless @success
+        IO.write("#{cache2_path}/update.log", Terraspace::Logger.logs)
+        return unless success
 
         sh "terraform state pull > #{cache2_path}/state.json"
         sh "terraform output -json > #{cache2_path}/output.json"
       end
     end
+
+    def cani?(exit_on_error: true)
+      return true unless Terraspace.cloud?
+      api.create_update(cani: 1, exit_on_error: exit_on_error)
+    end
   end
 end

data/lib/terraspace/cloud/{folder → upload}/base.rb

@@ -1,4 +1,4 @@
-class Terraspace::Cloud::Folder
+class Terraspace::Cloud::Upload
   class Base < Terraspace::Cloud::Base
     def initialize(options={})
       super

data/lib/terraspace/cloud/{folder → upload}/package.rb

@@ -1,6 +1,6 @@
 require 'zip_folder'
 
-class Terraspace::Cloud::Folder
+class Terraspace::Cloud::Upload
   class Package < Base
     def build
       copy

data/lib/terraspace/cloud/{folder → upload}/tidy.rb

@@ -1,4 +1,4 @@
-class Terraspace::Cloud::Folder
+class Terraspace::Cloud::Upload
   class Tidy < Base
     def cleanup
       removals.each do |removal|

data/lib/terraspace/cloud/upload.rb

@@ -0,0 +1,53 @@
+module Terraspace::Cloud
+  class Upload < Base
+    def create(data) # data: {stream_id:, type: }
+      zip_path = Package.new(@options.merge(data)).build
+      upload = create_upload(data)
+      upload_project(upload, zip_path)
+      upload
+    end
+
+  private
+    def create_upload(params)
+      upload = api.create_upload(params)
+      if errors?(upload)
+        error_message(upload)
+        exit 1 # Consider: raise exception can rescue higher up
+      else
+        upload
+      end
+    end
+
+    def upload_project(upload, path, retries=0)
+      url = upload['data']['attributes']['url']
+      uri = URI.parse(url)
+      object_content = IO.read(path)
+      sleep 2
+      resp = Net::HTTP.start(uri.host) do |http|
+        http.send_request(
+          'PUT',
+          uri.request_uri,
+          object_content,
+          'content-type' => ''
+        )
+      end
+      return if resp.code =~ /^20/
+
+      log_errors(resp.body, :debug)
+      if resp.body.include?('expired')
+        if retries >= 10 # 15m * 10 = 150m = 2.5h
+          log_errors(resp.body, :info)
+          exit 1
+        end
+        logger.debug("Retrying upload")
+        upload = create_upload(upload_id: upload['data']['id'])
+        upload_project(upload, path, retries+1)
+      end
+    end
+
+    def log_errors(body, level=:debug)
+      logger.send(level, "ERROR: Uploader uploading code")
+      logger.send(level, "resp body #{body}")
+    end
+  end
+end

data/lib/terraspace/cloud/vcs/base.rb

@@ -0,0 +1,6 @@
+class Terraspace::Cloud::Vcs
+  class Base
+    extend Memoist
+    include Terraspace::Util::Logging
+  end
+end