terraspace 0.2.1 → 0.3.1

data/lib/terraspace/terraform/cloud/sync.rb

@@ -0,0 +1,41 @@
+module Terraspace::Terraform::Cloud
+  class Sync < Terraspace::CLI::Base
+    extend Memoist
+    include Terraspace::Terraform::Api::Client
+
+    # Note about why workspace.create is called:
+    #
+    # CLI::Init#run
+    #   init => runs `terraform init`
+    #   build_remote_dependencies
+    #   sync_cloud => leads to create_workspace
+    #
+    # The `terraform init` will auto-create the TFC workspace
+    # If there is a .terraform folder the config.init.mode == "auto" though,
+    # then the workspace won't be created.
+    # So we check and create the workspace if necessary.
+    def run
+      # Note: workspace still gets created by `terraform init` However, variables wont be sync if returns early
+      return unless Terraspace.config.cloud.auto_sync || @options[:override_auto_sync]
+      return unless workspaces_backend?
+      logger.info "Syncing to Terraform Cloud: #{@mod.name} => #{workspace_name}"
+      @api = Terraspace::Terraform::Api.new(@mod, remote)
+      workspace.create_or_update
+      workspace.set_working_dir
+      workspace.set_env_vars
+    end
+
+    def workspace
+      @api.workspace
+    end
+
+    def workspaces_backend?
+      remote && remote['workspaces']
+    end
+
+    # already memoized in Api::Client
+    def backend
+      Terraspace::Compiler::Backend::Parser.new(@mod).result
+    end
+  end
+end

data/lib/terraspace/terraform/cloud/syncer.rb

@@ -0,0 +1,52 @@
+module Terraspace::Terraform::Cloud
+  class Syncer < Terraspace::CLI::Base
+    extend Memoist
+    include Terraspace::Compiler::DirsConcern
+    include Terraspace::Util::Sure
+
+    def run
+      are_you_sure?
+      mods.each do |mod|
+        run_sync(mod)
+      end
+    end
+
+    def mods
+      mod = @options[:mod]
+      mod ? [mod] : stack_names
+    end
+
+    def run_sync(mod)
+      sync(mod).run
+    end
+
+    def sync(mod)
+      Sync.new(@options.merge(mod: mod))
+    end
+    memoize :sync
+
+    def are_you_sure?
+      message =<<~EOL
+        About to sync these project stacks with Terraform Cloud workspaces:
+
+            Stack => Workspace
+      EOL
+
+      mods.each do |mod|
+        sync = sync(mod)
+        message << "    #{mod} => #{sync.workspace_name}\n"
+      end
+      message << <<~EOL
+
+        A sync does the following for each workspace:
+
+          1. Create or update workspace, including the VCS settings.
+          2. Set the working dir.
+          3. Set env and terraform variables.
+
+        Are you sure?
+      EOL
+      sure?(message.chop)
+    end
+  end
+end

data/lib/terraspace/terraform/cloud/workspace.rb

@@ -1,8 +1,9 @@
-class Terraspace::Terraform::Cloud
+module Terraspace::Terraform::Cloud
   class Workspace < Terraspace::CLI::Base
     extend Memoist
     include Terraspace::Util::Logging
     include Terraspace::Terraform::Api::Client
+    include Terraspace::Terraform::Api::Http::Concern
 
     # List will not have @mod set.
     def list
@@ -32,10 +33,15 @@ class Terraspace::Terraform::Cloud
       Terraspace::CLI::Init.new(@options.merge(calling_command: "cloud-setup")).run
     end
 
+    def create
+      build
+      return unless api
+      api.workspace.create
+    end
+
     def destroy
       build
-      return unless backend.dig('remote','workspaces') # in case called by terraspace down demo -y --destroy-workspace with a non-remote backend
-      api = Terraspace::Terraform::Api.new(@mod, remote)
+      return unless api
       workspace = api.workspace(exit_on_fail: false)
       unless workspace
         logger.info "Workspace #{workspace_name} not found for #{@mod.type}: #{@mod.name}"
@@ -43,25 +49,8 @@ class Terraspace::Terraform::Cloud
       end
       sure?
       logger.info "Destroying workspace #{workspace_name}"
-      api.destroy_workspace
-    end
-
-    def build
-      Terraspace::Builder.new(@options).run
-    end
-
-    def workspace_name
-      remote['workspaces']['name']
-    end
-
-    def remote
-      backend["remote"]
-    end
-
-    def backend
-      Terraspace::Compiler::Backend::Parser.new(@mod).result
+      api.workspace.destroy
     end
-    memoize :backend
 
     def sure?
       message = <<~EOL.chop + " " # chop to remove newline

data/lib/terraspace/terraform/hooks/builder.rb

@@ -34,7 +34,7 @@ module Terraspace::Terraform::Hooks
       exit_on_fail = exit_on_fail.nil? ? true : exit_on_fail
 
       logger.info "Running #{type} hook"
-      sh(execute, exit_on_fail: exit_on_fail)
+      Terraspace::Shell.new(@mod, execute, exit_on_fail: exit_on_fail).run
     end
   end
 end

data/lib/terraspace/terraform/remote_state/fetcher.rb

@@ -0,0 +1,122 @@
+module Terraspace::Terraform::RemoteState
+  class Fetcher
+    extend Memoist
+    include Terraspace::Util::Logging
+
+    def initialize(parent, identifier, options={})
+      @parent, @identifier, @options = parent, identifier, options
+      child_name, @output_key = identifier.split('.')
+      @child = Terraspace::Mod.new(child_name)
+    end
+
+    def run
+      validate!
+      pull
+      load
+    end
+
+    def output
+      run
+      if pull_success?
+        value = output_value
+        error = output_error(:key_not_found) unless @outputs.key?(@output_key)
+        OutputProxy.new(value, @options.merge(error: error))
+      else
+        error = output_error(:state_not_found)
+        OutputProxy.new(nil, @options.merge(error: error))
+      end
+    end
+
+    def output_value
+      return unless @outputs.key?(@output_key)
+      result = @outputs.dig(@output_key)
+      result.dig("value") if result
+    end
+
+    def output_error(type)
+      msg = case type
+      when :key_not_found
+        "Output #{@output_key} was not found for the #{@parent.name} tfvars file. Either #{@child.name} stack has not been deployed yet or it does not have this output: #{@output_key}"
+      when :state_not_found
+        "Output #{@output_key} could not be looked up for the #{@parent.name} tfvars file. #{@child.name} stack needs to be deployed"
+      end
+      msg = "(#{msg})"
+      log_message(msg)
+      msg
+    end
+
+    @@pull_successes = {}
+    @@download_shown = false
+    def pull
+      return if @@pull_successes[cache_key]
+      logger.info "Downloading tfstate files for dependencies defined in tfvars..." unless @@download_shown || @options[:quiet]
+      @@download_shown = true
+      logger.debug "Downloading tfstate for stack: #{@child.name}"
+      Terraspace::CLI::Init.new(mod: @child.name, calling_command: "apply", quiet: true).init # init not run, so only init
+      FileUtils.mkdir_p(File.dirname(state_path))
+      command = "cd #{@child.cache_dir} && terraform state pull > #{state_path}"
+      logger.debug "=> #{command}"
+      success = system(command)
+      # Can error if using a old terraform version and the statefile was created with a newer version of terraform
+      # IE: Failed to refresh state: state snapshot was created by Terraform v0.13.2, which is newer than current v0.12.29;
+      #     upgrade to Terraform v0.13.2 or greater to work with this state
+      unless success
+        logger.info "Error running: #{command}".color(:red)
+        logger.info "Please fix the error before continuing"
+      end
+      @@pull_successes[cache_key] = success
+    end
+
+    def load
+      return self unless pull_success?
+
+      # use or set cache
+      if @@cache[cache_key]
+        @outputs = @@cache[cache_key]
+      else
+        @outputs = @@cache[cache_key] = read_statefile_outputs
+      end
+
+      self
+    end
+    memoize :load
+
+    def cache_key
+      @child.name
+    end
+
+    def read_statefile_outputs
+      data = JSON.load(IO.read(state_path))
+      data ? data['outputs'] : {}
+    end
+
+    def pull_success?
+      @@pull_successes[cache_key]
+    end
+
+    def state_path
+      "#{Terraspace.tmp_root}/remote_state/#{@child.build_dir}/state.json"
+    end
+
+    # Note we already validate mod exist at the terraform_output helper. This is just in case that logic changes.
+    def validate!
+      return if @child.exist?
+      logger.error "ERROR: stack #{@child.name} not found".color(:red)
+      exit 1
+    end
+
+    # Using debug level because all the tfvar files always get evaluated.
+    # So dont want these messages to show up and be noisy unless debugging.
+    def log_message(msg)
+      logger.debug "DEBUG: #{msg}".color(:yellow)
+    end
+
+    cattr_accessor :cache, default: {}
+    class << self
+      def flush!
+        @@pull_successes = {}
+        @@cache = {}
+      end
+    end
+  end
+end

data/lib/terraspace/terraform/remote_state/marker/output.rb

@@ -0,0 +1,39 @@
+module Terraspace::Terraform::RemoteState::Marker
+  class Output
+    include Terraspace::Util::Logging
+
+    def initialize(mod, identifier, options={})
+      @mod, @identifier, @options = mod, identifier, options
+      @parent_name = @mod.name
+      @child_name, @output_key = @identifier.split('.')
+    end
+
+    def build
+      if valid?
+        Terraspace::Dependency::Registry.register(@parent_name, @child_name)
+      else
+        warning
+      end
+      # MARKER for debugging. Only appears on 1st pass. Will not see unless changing Terraspace code for debugging.
+      marker = "MARKER:terraform_output('#{@identifier}')"
+      Terraspace::Terraform::RemoteState::OutputProxy.new(marker, @options)
+    end
+
+    def valid?
+      self.class.stack_names.include?(@child_name)
+    end
+
+    def warning
+      logger.warn "WARN: The #{@child_name} stack does not exist".color(:yellow)
+      caller_line = caller.find { |l| l.include?('.tfvars') }
+      source_code = PrettyTracer.new(caller_line).source_code
+      logger.info source_code
+    end
+
+    class << self
+      extend Memoist
+      # Marker::Output uses DirsConcern stack_names to check if stacks are valid
+      include Terraspace::Compiler::DirsConcern
+    end
+  end
+end

data/lib/terraspace/terraform/remote_state/marker/pretty_tracer.rb

@@ -0,0 +1,37 @@
+module Terraspace::Terraform::RemoteState::Marker
+  class PrettyTracer
+    def initialize(caller_line)
+      @caller_line = caller_line
+    end
+
+    # /full/path/to/app/stacks/a1/tfvars/dev.tfvars:4:in `__tilt_5560'
+    def source_code
+      line = @caller_line.sub(/:in `.*/,'')
+      path, error_line_number = line.split(':')
+      pretty_trace(path, error_line_number.to_i)
+    end
+
+    def pretty_trace(path, error_line_number)
+      io = StringIO.new
+      context = 5 # lines of context
+      top, bottom = [error_line_number-context-1, 0].max, error_line_number+context-1
+
+      io.puts "Here's the line in #{Terraspace::Util.pretty_path(path)} with the error:\n\n"
+
+      lines = IO.read(path).split("\n")
+      context = 5 # lines of context
+      top, bottom = [error_line_number-context-1, 0].max, error_line_number+context-1
+      spacing = lines.size.to_s.size
+      lines[top..bottom].each_with_index do |line_content, index|
+        line_number = top+index+1
+        if line_number == error_line_number
+          io.printf("%#{spacing}d %s\n".color(:red), line_number, line_content)
+        else
+          io.printf("%#{spacing}d %s\n", line_number, line_content)
+        end
+      end
+
+      io.string
+    end
+  end
+end

data/lib/terraspace/terraform/remote_state/output_proxy.rb

@@ -0,0 +1,29 @@
+module Terraspace::Terraform::RemoteState
+  class OutputProxy
+    # raw: can be anything: String, Array, Hash, etc
+    # options: original options passed by user with terraform_output
+    attr_reader :raw, :options
+    def initialize(raw, options={})
+      @raw, @options = raw, options
+      @format = @options[:format]
+    end
+
+    # Should always return a String
+    def to_s
+      case @format
+      when "string"
+        content.to_s
+      else # "json"
+        content.to_json
+      end
+    end
+
+    def content
+      if @raw.nil?
+        @options[:mock] || @options[:error]
+      else
+        @raw
+      end
+    end
+  end
+end

data/lib/terraspace/terraform/runner.rb

@@ -7,6 +7,7 @@ module Terraspace::Terraform
     def initialize(name, options={})
       @name = name
       super(options)
+      @retries = 1
     end
 
     def run
@@ -21,17 +22,37 @@ module Terraspace::Terraform
       params = args.flatten.join(' ')
       command = "terraform #{name} #{params}"
       run_hooks(name) do
-        sh(command, env: custom.env_vars)
+        Terraspace::Shell.new(@mod, command, @options.merge(env: custom.env_vars)).run
+      end
+    rescue Terraspace::InitRequiredError => e
+      logger.info "Terraform reinitialization required detected. Will run `terraform init` and try again."
+      logger.debug "Retry attempt: #{@retries}"
+      logger.debug "#{e.class}"
+      Runner.new("init", @options).run
+      if @retries <= 3
+        backoff = 2 ** @retries # 2, 4, 8
+        logger.debug "Waiting #{backoff}s before retrying"
+        sleep(backoff)
+        @retries += 1
+        retry
+      else
+        logger.info "ERROR: #{e.message}"
+        exit 1
       end
     end
 
     @@current_dir_message_shown = false
     def current_dir_message
       return if @@current_dir_message_shown
-      logger.info "Current directory: #{Terraspace::Util.pretty_path(@mod.cache_dir)}"
+      log "Current directory: #{Terraspace::Util.pretty_path(@mod.cache_dir)}"
       @@current_dir_message_shown = true
     end
 
+    def log(msg)
+      # quiet useful for RemoteState::Fetcher
+      @options[:quiet] ? logger.debug(msg) : logger.info(msg)
+    end
+
     def run_hooks(name, &block)
       hooks = Hooks::Builder.new(@mod, name)
       hooks.build # build hooks
@@ -59,18 +80,7 @@ module Terraspace::Terraform
       yield
       t2 = Time.now
       if %w[apply destroy].include?(@name)
-        puts "Time took: #{pretty_time(t2-t1)}"
-      end
-    end
-
-    # http://stackoverflow.com/questions/4175733/convert-duration-to-hoursminutesseconds-or-similar-in-rails-3-or-ruby
-    def pretty_time(total_seconds)
-      minutes = (total_seconds / 60) % 60
-      seconds = total_seconds % 60
-      if total_seconds < 60
-        "#{seconds.to_i}s"
-      else
-        "#{minutes.to_i}m #{seconds.to_i}s"
+        logger.info "Time took: #{pretty_time(t2-t1)}"
       end
     end
   end