terraspace 0.2.4 → 0.3.4

data/lib/terraspace/core.rb

@@ -27,6 +27,10 @@ module Terraspace
     end
     memoize :tmp_root
 
+    def log_root
+      "#{root}/log"
+    end
+
     def configure(&block)
       App.instance.configure(&block)
     end
@@ -38,10 +42,15 @@ module Terraspace
     end
     memoize :config
 
+    @@logger = nil
     def logger
-      config.logger
+      @@logger ||= config.logger
+    end
+
+    # allow different logger when running up all
+    def logger=(v)
+      @@logger = v
     end
-    memoize :logger
 
     def check_project!
       return if File.exist?("#{Terraspace.root}/config/app.rb")

data/lib/terraspace/dependency/graph.rb

@@ -0,0 +1,140 @@
+module Terraspace::Dependency
+  class Graph
+    include Terraspace::Util::Logging
+
+    attr_reader :nodes
+    def initialize(stack_names, dependencies, options={})
+      @stack_names, @dependencies, @options = stack_names, dependencies, options
+      @nodes = []
+      @batches = []
+    end
+
+    def build
+      precreate_all_nodes
+      build_nodes_with_dependencies # @nodes has dependency graph info afterwards
+      check_circular_dependencies!
+      @nodes = filter_nodes
+      check_empty_nodes!
+      build_batches
+      clean_batches
+      @batches
+    end
+
+    # Only check when stacks option is pass. Edge case: There can be app/modules but no app/stacks yet
+    def check_empty_nodes!
+      return unless @nodes.empty? && @options[:stacks]
+      logger.error "ERROR: No stacks were found that match: #{@options[:stacks].join(' ')}".color(:red)
+      exit 1
+    end
+
+    def check_circular_dependencies!
+      @nodes.each do |node|
+        check_cycle(node)
+      end
+    end
+
+    MAX_CYCLE_DEPTH = Integer(ENV['TS_MAX_CYCLE_DEPTH'] || 100)
+    def check_cycle(node, depth=0, list=[])
+      if depth > MAX_CYCLE_DEPTH
+        logger.error "ERROR: It seems like there is a circular dependency! Stacks involved: #{list.uniq}".color(:red)
+        exit 1
+      end
+      node.parents.each do |parent|
+        check_cycle(parent, depth+1, list += [parent])
+      end
+    end
+
+    def precreate_all_nodes
+      @stack_names.each do |name|
+        node = Node.find_or_create_by(name: name)
+        save_node(node)
+      end
+    end
+
+    def build_nodes_with_dependencies
+      @dependencies.each do |item|
+        parent_name, child_name = item.split(':')
+        save_node_parent(parent_name, child_name)
+      end
+    end
+
+    def save_node_parent(parent_name, child_name)
+      parent = Node.find_by(name: parent_name)
+      child = Node.find_by(name: child_name)
+      child.parent!(parent)
+      save_node(parent)
+      save_node(child)
+    end
+
+    def build_batches
+      @batches[0] = Set.new(leaves)
+      leaves.each do |leaf|
+        leaf.parents.each do |parent|
+          build_batch(parent)
+        end
+      end
+    end
+
+    # So stack nodes dont get deployed more than once and too early
+    def clean_batches
+      all = Set.new
+      # batch is a set
+      @batches.reverse.each do |batch|
+        batch.each do |node|
+          batch.delete(node) if all.include?(node)
+        end
+        all += batch
+      end
+      @batches.reject! { |batch| batch.empty? }
+      @batches
+    end
+
+    def build_batch(leaf, depth=1)
+      @batches[depth] ||= Set.new
+      @batches[depth] << leaf
+      leaf.parents.each do |parent|
+        build_batch(parent, depth+1)
+      end
+    end
+
+    def filter_nodes
+      @filtered = []
+      top_nodes.each { |node| apply_filter(node) }
+      # draw_full_graph option is only used internally by All::Grapher
+      update_parents! unless @options[:draw_full_graph]
+      @options[:draw_full_graph] ? @nodes : @filtered
+    end
+
+    # remove missing parents references since they will be filtered out
+    def update_parents!
+      @filtered.each do |node|
+        new_parents = node.parents & @filtered
+        node.parents = new_parents
+      end
+    end
+
+    def apply_filter(parent, keep=false)
+      keep ||= @options[:stacks].blank?
+      keep ||= @options[:stacks].include?(parent.name)  # apply filter
+      if keep
+        parent.filtered = true
+        @filtered << parent
+      end
+      parent.children.sort_by(&:name).each do |child|
+        apply_filter(child, keep)
+      end
+    end
+
+    def leaves
+      @nodes.select { |n| n.children.empty? }.sort_by(&:name)
+    end
+
+    def top_nodes
+      @nodes.select { |n| n.parents.empty? }.sort_by(&:name)
+    end
+
+    def save_node(node)
+      @nodes << node unless @nodes.detect { |n| n.name == node.name }
+    end
+  end
+end

data/lib/terraspace/dependency/node.rb

@@ -0,0 +1,38 @@
+module Terraspace::Dependency
+  class Node
+    attr_reader :name
+    attr_accessor :children, :parents, :filtered
+    def initialize(name)
+      @name = name
+      @children, @parents = Set.new, Set.new
+    end
+
+    def highlighted?
+      @filtered
+    end
+
+    def inspect
+      @name
+    end
+
+    def parent!(parent)
+      @parents << parent
+      parent.children << self
+    end
+
+    class << self
+      @@nodes = []
+      def find_or_create_by(name:)
+        node = find_by(name: name)
+        return node if node
+        node = Node.new(name)
+        @@nodes << node
+        node
+      end
+
+      def find_by(name:)
+        @@nodes.find { |n| n.name == name }
+      end
+    end
+  end
+end

data/lib/terraspace/dependency/registry.rb

@@ -0,0 +1,11 @@
+module Terraspace::Dependency
+  class Registry
+    cattr_accessor :data, default: Set.new
+
+    class << self
+      def register(parent_name, child_name)
+        @@data << "#{parent_name}:#{child_name}"
+      end
+    end
+  end
+end

data/lib/terraspace/logger.rb

@@ -2,25 +2,13 @@ require 'logger'
 
 module Terraspace
   class Logger < ::Logger
-    # Only need to override the add method as the other calls all lead to it.
-    def add(severity, message = nil, progname = nil)
-      # Taken from Logger#add source
-      # https://ruby-doc.org/stdlib-2.5.1/libdoc/logger/rdoc/Logger.html#method-i-add
-      if message.nil?
-        if block_given?
-          message = yield
-        else
-          message = progname
-          progname = @progname
-        end
+    def format_message(severity, datetime, progname, msg)
+      line = if @logdev.dev == $stdout || @logdev.dev == $stderr
+        msg # super simple format if stdout
+      else
+        super # use the configured formatter
       end
-
-      super # original logic
-    end
-
-    # plain formatting
-    def format_message(severity, timestamp, progname, msg)
-      "#{msg}\n"
+      line =~ /\n$/ ? line : "#{line}\n"
     end
   end
 end

data/lib/terraspace/logger/formatter.rb

@@ -0,0 +1,13 @@
+class Terraspace::Logger
+  class Formatter < ::Logger::Formatter
+    def call(severity, time, progname, msg)
+      # careful changing the format. All::Summary uses a regexp on this format to remove the timestamp
+      "[#{format_datetime(time)} ##{Process.pid} #{progname}]: #{msg}"
+    end
+
+  private
+    def format_datetime(time)
+      time.strftime("%Y-%m-%dT%H:%M:%S")
+    end
+  end
+end

data/lib/terraspace/mod.rb

@@ -10,6 +10,7 @@ module Terraspace
     include Terraspace::Util
 
     attr_reader :name, :consider_stacks, :instance, :options
+    attr_accessor :resolved # dependencies resolved
     def initialize(name, options={})
       @name, @options = placeholder(name), options
       @consider_stacks = options[:consider_stacks].nil? ? true : options[:consider_stacks]
@@ -74,6 +75,10 @@ module Terraspace
       end
     end
 
+    def exist?
+      !!root
+    end
+
     # Relative folder path without app or vendor. For example, the actual location can be found in a couple of places
     #
     #     app/modules/vpc
@@ -86,8 +91,9 @@ module Terraspace
     #     modules/vpc
     #
     def build_dir(disable_instance: false)
-      if !disable_instance && !@instance.nil?
+      if !@instance.nil? && type_dir == "stacks" && !disable_instance
         # add _ in front so instance doesnt collide with other default stacks
+        # never add for app/modules sources
         instance_name = [name, @instance].compact.join('.')
       else
         instance_name = name

data/lib/terraspace/plugin/summary/interface.rb

@@ -18,7 +18,7 @@ module Terraspace::Plugin::Summary
       # Note: will not change any of these instance variables unless we note breaking changes
       @bucket = @info[bucket_field]
       @key = @info[key_field] # key_field is INTERFACE METHOD IE: aws: key , google: prefix
-      @folder = folder(@key)
+      @folder = folder(@key)  # useful as prefix for performance when listing objects in buckets
       @dest = dest(@bucket)
       # May change any of these instance variables that follow
       @dest_folder = "#{@dest}/#{@folder}"

data/lib/terraspace/seeder/where.rb

@@ -20,11 +20,15 @@ class Terraspace::Seeder
     end
 
     def app_path
-      "#{Terraspace.root}/app/#{@mod.build_dir}/tfvars/#{Terraspace.env}.tfvars"
+      "#{Terraspace.root}/app/#{@mod.build_dir}/tfvars/#{seed_file}.tfvars"
     end
 
     def seed_path
-      "#{Terraspace.root}/seed/tfvars/#{@mod.build_dir}/#{Terraspace.env}.tfvars"
+      "#{Terraspace.root}/seed/tfvars/#{@mod.build_dir}/#{seed_file}.tfvars"
+    end
+
+    def seed_file
+      @options[:instance] || Terraspace.env
     end
   end
 end

data/lib/terraspace/shell.rb

@@ -0,0 +1,107 @@
+require 'open3'
+
+module Terraspace
+  class Shell
+    include Util::Logging
+
+    def initialize(mod, command, options={})
+      @mod, @command, @options = mod, command, options
+      @error_type, @error_messages = nil, ''
+    end
+
+    # requires @mod to be set
+    # quiet useful for RemoteState::Fetcher
+    def run
+      msg = "=> #{@command}"
+      @options[:quiet] ? logger.debug(msg) : logger.info(msg)
+      return if ENV['TS_TEST']
+      shell
+    end
+
+    def shell
+      env = @options[:env] || {}
+      env.stringify_keys!
+      if @options[:shell] == "system" # terraspace console
+        system(env, @command, chdir: @mod.cache_dir)
+      else
+        popen3(env)
+      end
+    end
+
+    def popen3(env)
+      Open3.popen3(env, @command, chdir: @mod.cache_dir) do |stdin, stdout, stderr, wait_thread|
+        mimic_terraform_input(stdin, stdout)
+        while err = stderr.gets
+          @error_type ||= known_error_type(err)
+          if @error_type
+            @error_messages << err
+          else
+            # Sometimes may print a "\e[31m\n" which like during dependencies fetcher init
+            # suppress it so dont get a bunch of annoying "newlines"
+            next if err == "\e[31m\n" && @options[:suppress_error_color]
+            logger.error(err)
+          end
+        end
+
+        status = wait_thread.value.exitstatus
+        exit_status(status)
+      end
+    end
+
+    def known_error_type(err)
+      if reinitialization_required?(err)
+        :reinitialization_required
+      elsif bucket_not_found?(err)
+        :bucket_not_found
+      end
+    end
+
+    def bucket_not_found?(err)
+      # Message is included in aws, azurerm, and google. See: https://bit.ly/3iOKDri
+      err.include?("Failed to get existing workspaces")
+    end
+
+    def reinitialization_required?(err)
+      err.include?("reinitialization required") ||
+      err.include?("terraform init") ||
+      err.include?("require reinitialization")
+    end
+
+    def exit_status(status)
+      return if status == 0
+
+      exit_on_fail = @options[:exit_on_fail].nil? ? true : @options[:exit_on_fail]
+      if @error_type == :reinitialization_required
+        raise InitRequiredError.new(@error_messages)
+      elsif @error_type == :bucket_not_found
+        raise BucketNotFoundError.new(@error_messages)
+      elsif exit_on_fail
+        logger.error "Error running command: #{@command}".color(:red)
+        exit status
+      end
+    end
+
+    # Terraform doesnt seem to stream the line that prompts with "Enter a value:" when using Open3.popen3
+    # Hack around it by mimicking the "Enter a value:" prompt
+    #
+    # Note: system does stream the prompt but using Open3.popen3 so we can capture output to save to logs.
+    def mimic_terraform_input(stdin, stdout)
+      shown = false
+      patterns = [
+        "Only 'yes' will be accepted", # prompt for apply. can happen on apply
+        "\e[0m\e[1mvar.", # prompts for variable input. can happen on plan or apply. looking for bold marker also in case "var." shows up somewhere else
+      ]
+      while out = stdout.gets
+        logger.info(out) unless shown && out.include?("Enter a value:")
+        shown = false if out.include?("Enter a value:") # reset shown in case of multiple input prompts
+
+        # Sometimes stdout doesnt flush and show "Enter a value: ", so mimic it
+        if patterns.any? { |pattern| out.include?(pattern) }
+          print "  Enter a value: ".bright
+          shown = true
+          stdin.write_nonblock($stdin.gets)
+        end
+      end
+    end
+  end
+end

data/lib/terraspace/terraform/api.rb

@@ -1,58 +1,20 @@
 module Terraspace::Terraform
   class Api
     extend Memoist
-    include Client
-    include Terraspace::Util::Logging
 
     def initialize(mod, remote)
-      @mod = mod
-      @organization = remote['organization']
-      @workspace_name = remote['workspaces']['name']
+      @mod, @remote = mod, remote
     end
 
-    # Docs: https://www.terraform.io/docs/cloud/api/workspaces.html
-    def set_working_dir
-      return if working_directory == workspace['attributes']['working-directory']
-
-      payload = {
-        data: {
-          attributes: {
-            "working-directory": working_directory
-          },
-          type: "workspaces"
-        }
-      }
-      http.patch("organizations/#{@organization}/workspaces/#{@workspace_name}", payload)
-    end
-
-    def working_directory
-      cache_dir = @mod.cache_dir.sub("#{Terraspace.root}/", '')
-      relative_root = Terraspace.config.cloud.relative_root # prepended to TFC Working Directory
-      relative_root ? "#{relative_root}/#{cache_dir}" : cache_dir
-    end
-
-    def set_env_vars
-      Vars.new(@mod, workspace).run
-    end
-
-    def workspace(options={})
-      payload = http.get("organizations/#{@organization}/workspaces/#{@workspace_name}")
-      # Note only way to get here is to bypass init. Example:
-      #
-      #     terraspace up demo --no-init
-      #
-      unless payload || options[:exit_on_fail] == false
-        logger.error "ERROR: Unable to find the workspace. The workspace may not exist. Or the Terraform token may be invalid. Please double check your Terraform token.".color(:red)
-        exit 1
-      end
-      return unless payload
-      payload['data']
+    def workspace
+      Workspace.new(@mod, @remote['organization'], @remote['workspaces']['name'])
     end
     memoize :workspace
 
-    def destroy_workspace
-      # resp payload from delete operation is nil
-      http.delete("/organizations/#{@organization}/workspaces/#{@workspace_name}")
+    def runs
+      workspace_id = workspace.details['id']
+      Runs.new(workspace_id)
     end
+    memoize :runs
   end
 end