terraspace 0.2.0 → 0.3.0

data/lib/terraspace/cli/log/concern.rb

@@ -0,0 +1,24 @@
+class Terraspace::CLI::Log
+  module Concern
+    # Filters for lines that belong to the last ran process pid
+    def readlines(path)
+      lines = IO.readlines(path)
+      found = lines.reverse.find do |line|
+        pid(line) # search in reverse order for lines with interesting info
+      end
+      unless found
+        puts "WARN: Could not find the pid in the logfile #{Terraspace::Util.pretty_path(path)}".color(:yellow)
+        return []
+      end
+
+      pid = pid(found)
+      lines.select {|l| l.include?(" ##{pid} ") }
+    end
+
+    # [2020-09-06T21:58:25 #11313 terraspace up b1]:
+    def pid(line)
+      md = line.match(/:\d{2} #(\d+) /)
+      md[1] if md
+    end
+  end
+end

data/lib/terraspace/cli/logs.rb

@@ -0,0 +1,15 @@
+class Terraspace::CLI
+  class Logs < Terraspace::Command
+    desc "truncate", "Truncates logs. IE: Removes contents and zero bytes the files"
+    long_desc Help.text("logs/truncate")
+    def truncate
+      Tasks.new(options).truncate
+    end
+
+    desc "remove", "Removes logs"
+    long_desc Help.text("logs/remove")
+    def remove
+      Tasks.new(options).remove
+    end
+  end
+end

data/lib/terraspace/cli/logs/tasks.rb

@@ -0,0 +1,32 @@
+class Terraspace::CLI::Logs
+  class Tasks
+    def initialize(options={})
+      @options = options
+    end
+
+    def truncate
+      puts "Truncating log files in #{pretty_log_root}/" unless @options[:mute]
+      log_files.each do |path|
+        File.open(path, "w").close # truncates files
+      end
+    end
+
+    def remove
+      puts "Removing all files in #{pretty_log_root}/" unless @options[:mute]
+      FileUtils.rm_rf(log_root)
+      FileUtils.mkdir_p(log_root)
+    end
+
+    def log_files
+      Dir.glob("#{log_root}/**/*.log")
+    end
+
+    def pretty_log_root
+      Terraspace::Util.pretty_path(log_root)
+    end
+
+    def log_root
+      Terraspace.config.log.root
+    end
+  end
+end

data/lib/terraspace/cli/new/git_hook.rb

@@ -25,7 +25,7 @@ class Terraspace::CLI::New
     def terraspace_build_commands
       code = []
       @options[:envs].each do |env|
-        code << %Q|TS_ENV=#{env} terraspace build placeholder|
+        code << %Q|TS_ENV=#{env} terraspace build|
       end
       code.join("\n")
     end

data/lib/terraspace/cli/summary.rb

@@ -16,7 +16,7 @@ class Terraspace::CLI
       puts "Summary of resources based on backend storage statefiles"
       backend_expr = '.terraspace-cache/**/backend.*'
       # Currently summary assumes backend are within the same bucket and key prefix
-      backend = Dir.glob(backend_expr).first
+      backend = Dir.glob(backend_expr).find { |p| p.include?("/#{Terraspace.env}/") }
       process(backend) if backend
     end
 

data/lib/terraspace/cli/tfc_concern.rb

@@ -0,0 +1,14 @@
+class Terraspace::CLI
+  module TfcConcern
+    extend Memoist
+
+    def tfc?
+      !!backend["remote"]
+    end
+
+    def backend
+      Terraspace::Compiler::Backend::Parser.new(@mod).result
+    end
+    memoize :backend
+  end
+end

data/lib/terraspace/cli/up.rb

@@ -0,0 +1,32 @@
+class Terraspace::CLI
+  class Up < Base
+    include TfcConcern
+
+    def run
+      build
+      if @options[:yes] && !tfc?
+        plan
+        Commander.new("apply", @options.merge(plan: plan_path)).run
+      else
+        Commander.new("apply", @options).run
+      end
+    end
+
+  private
+    # must build to compute tfc?
+    def build
+      Terraspace::Builder.new(@options).run
+      @options[:build] = false
+    end
+
+    def plan
+      FileUtils.mkdir_p(File.dirname(plan_path))
+      Commander.new("plan", @options.merge(out: plan_path)).run
+    end
+
+    def plan_path
+      @@timestamp ||= Time.now.utc.strftime("%Y%m%d%H%M%S")
+      "#{Terraspace.tmp_root}/plans/#{@mod.name}-#{@@timestamp}.plan"
+    end
+  end
+end

data/lib/terraspace/compiler/builder.rb

@@ -8,14 +8,14 @@ module Terraspace::Compiler
 
     def build
       build_config
-      build_module
+      build_module if @mod.resolved
       build_tfvars
     end
 
     # build common config files: provider and backend for the root module
     def build_config
       return unless build?
-      build_config_templates
+      build_config_terraform
     end
 
     def build_module
@@ -34,7 +34,7 @@ module Terraspace::Compiler
       @mod.type == "stack" || @mod.root_module?
     end
 
-    def build_config_templates
+    def build_config_terraform
       expr = "#{Terraspace.root}/config/terraform/**/*"
       Dir.glob(expr).each do |path|
         next unless File.file?(path)

data/lib/terraspace/compiler/cleaner.rb

@@ -5,7 +5,7 @@ module Terraspace::Compiler
     end
 
     def clean
-      return if ENV['TS_NO_CLEAN']
+      return if ENV['TS_CLEAN'] == '0'
       backend_change_purge
       remove_materialized_artifacts
       # remove_materialized_artifacts_dot_terraform

data/lib/terraspace/compiler/cleaner/backend_change.rb

@@ -9,14 +9,9 @@ class Terraspace::Compiler::Cleaner
     def purge
       return unless purge?
 
+      are_you_sure? if local_statefile_exist?
       cache_root = Terraspace::Util.pretty_path(Terraspace.cache_root)
-      message =<<~EOL
-        Backend change detected. Will remove #{cache_root} for complete reinitialization
-        WARN: If you are using local storage for state, this will remove it.
-        Will remove #{cache_root}
-      EOL
-      sure?(message.strip)
-      logger.info "Backend change detected. Removing #{cache_root} for complete reinitialization"
+      logger.debug "Backend change detected. Removing #{cache_root} for complete reinitialization"
       FileUtils.rm_rf(Terraspace.cache_root)
     end
 
@@ -26,6 +21,15 @@ class Terraspace::Compiler::Cleaner
       current_backend != fresh_backend
     end
 
+    def local_statefile_exist?
+      # Note: Will not go into .terraform folders. No need to for terraform.tfstate
+      Dir.glob("#{Terraspace.cache_root}/**/*").each do |path|
+        basename = File.basename(path)
+        return true if basename == 'terraform.tfstate'
+      end
+      false
+    end
+
     def current_backend
       materialized_path = find_src_path("#{@mod.cache_dir}/backend*")
       IO.read(materialized_path) if materialized_path
@@ -37,6 +41,16 @@ class Terraspace::Compiler::Cleaner
     end
 
   private
+    def are_you_sure?
+      cache_root = Terraspace::Util.pretty_path(Terraspace.cache_root)
+      message =<<~EOL
+        Backend change detected. Will remove #{cache_root} for complete reinitialization
+        #{"WARN: You are using local storage for state, this will remove it.".color(:yellow)}
+        Will remove #{cache_root} and all terraform.tfstate files
+      EOL
+      sure?(message.strip) # from Util
+    end
+
     def find_src_path(expr)
       path = Dir.glob(expr).first
       path if path && File.exist?(path)

data/lib/terraspace/compiler/dirs_concern.rb

@@ -0,0 +1,47 @@
+module Terraspace::Compiler
+  module DirsConcern
+    extend ActiveSupport::Concern
+    extend Memoist
+
+    def cache_dirs
+      cache_dirs = []
+      with_each_mod("stacks") do |mod|
+        cache_dirs << mod.cache_dir
+      end
+      cache_dirs
+    end
+
+    def with_each_mod(type_dir)
+      mod_names(type_dir).each do |mod_name|
+        consider_stacks = type_dir == "stacks"
+        mod = Terraspace::Mod.new(mod_name, @options.merge(consider_stacks: consider_stacks))
+        yield(mod)
+      end
+    end
+
+    def mod_names(type_dir)
+      names, built = [], []
+      local_paths(type_dir).each do |path|
+        next unless File.directory?(path)
+        mod_name = File.basename(path)
+        next if built.include?(mod_name) # ensures modules in app folder take higher precedence than vendor folder
+        names << mod_name
+      end
+      names
+    end
+    memoize :mod_names
+
+    def local_paths(type_dir)
+      dirs("app/#{type_dir}/*") + dirs("vendor/#{type_dir}/*")
+    end
+
+    def dirs(path)
+      Dir.glob("#{Terraspace.root}/#{path}")
+    end
+
+    def stack_names
+      mod_names("stacks") - Terraspace.config.all.ignore_stacks
+    end
+    memoize :stack_names
+  end
+end

data/lib/terraspace/compiler/dsl/syntax/helpers/common.rb

@@ -1,7 +1,8 @@
 module Terraspace::Compiler::Dsl::Syntax::Helpers
   module Common
     extend Memoist
-
+    Fetcher = Terraspace::Terraform::RemoteState::Fetcher
+    Marker = Terraspace::Terraform::RemoteState::Marker
     Meta = Terraspace::Compiler::Dsl::Meta
 
     def var
@@ -24,5 +25,29 @@ module Terraspace::Compiler::Dsl::Syntax::Helpers
       command = ["terraspace"] + args
       command.join(separator)
     end
+
+    def terraform_output(identifier, options={})
+      if @mod.resolved # dependencies have been resolved
+        Fetcher.new(@mod, identifier, options).output
+      else
+        Marker::Output.new(@mod, identifier, options).build
+      end
+    end
+
+    def depends_on(*child_names, **options)
+      child_names.flatten!
+      child_names.map do |child_name|
+        each_depends_on(child_name, options)
+      end.join("\n")
+    end
+
+    def each_depends_on(child_name, options={})
+      if @mod.resolved # dependencies have been resolved
+        # Note: A generated line is not really needed. Dependencies are stored in memory. Added to assist users with debugging
+        "# #{@mod.name} depends on #{child_name}"
+      else
+        Marker::Output.new(@mod, child_name, options).build
+      end
+    end
   end
 end

data/lib/terraspace/core.rb

@@ -27,6 +27,10 @@ module Terraspace
     end
     memoize :tmp_root
 
+    def log_root
+      "#{root}/log"
+    end
+
     def configure(&block)
       App.instance.configure(&block)
     end
@@ -38,10 +42,15 @@ module Terraspace
     end
     memoize :config
 
+    @@logger = nil
     def logger
-      config.logger
+      @@logger ||= config.logger
+    end
+
+    # allow different logger when running up all
+    def logger=(v)
+      @@logger = v
     end
-    memoize :logger
 
     def check_project!
       return if File.exist?("#{Terraspace.root}/config/app.rb")

data/lib/terraspace/dependency/graph.rb

@@ -0,0 +1,139 @@
+module Terraspace::Dependency
+  class Graph
+    include Terraspace::Util::Logging
+
+    attr_reader :nodes
+    def initialize(stack_names, dependencies, options={})
+      @stack_names, @dependencies, @options = stack_names, dependencies, options
+      @nodes = []
+      @batches = []
+    end
+
+    def build
+      precreate_all_nodes
+      build_nodes_with_dependencies # @nodes has dependency graph info afterwards
+      check_circular_dependencies!
+      @nodes = filter_nodes
+      check_empty_nodes!
+      build_batches
+      clean_batches
+      @batches
+    end
+
+    def check_empty_nodes!
+      return unless @nodes.empty?
+      logger.error "ERROR: No stacks were found that match: #{@options[:stacks].join(' ')}".color(:red)
+      exit 1
+    end
+
+    def check_circular_dependencies!
+      @nodes.each do |node|
+        check_cycle(node)
+      end
+    end
+
+    MAX_CYCLE_DEPTH = Integer(ENV['TS_MAX_CYCLE_DEPTH'] || 100)
+    def check_cycle(node, depth=0, list=[])
+      if depth > MAX_CYCLE_DEPTH
+        logger.error "ERROR: It seems like there is a circular dependency! Stacks involved: #{list.uniq}".color(:red)
+        exit 1
+      end
+      node.parents.each do |parent|
+        check_cycle(parent, depth+1, list += [parent])
+      end
+    end
+
+    def precreate_all_nodes
+      @stack_names.each do |name|
+        node = Node.find_or_create_by(name: name)
+        save_node(node)
+      end
+    end
+
+    def build_nodes_with_dependencies
+      @dependencies.each do |item|
+        parent_name, child_name = item.split(':')
+        save_node_parent(parent_name, child_name)
+      end
+    end
+
+    def save_node_parent(parent_name, child_name)
+      parent = Node.find_by(name: parent_name)
+      child = Node.find_by(name: child_name)
+      child.parent!(parent)
+      save_node(parent)
+      save_node(child)
+    end
+
+    def build_batches
+      @batches[0] = Set.new(leaves)
+      leaves.each do |leaf|
+        leaf.parents.each do |parent|
+          build_batch(parent)
+        end
+      end
+    end
+
+    # So stack nodes dont get deployed more than once and too early
+    def clean_batches
+      all = Set.new
+      # batch is a set
+      @batches.reverse.each do |batch|
+        batch.each do |node|
+          batch.delete(node) if all.include?(node)
+        end
+        all += batch
+      end
+      @batches.reject! { |batch| batch.empty? }
+      @batches
+    end
+
+    def build_batch(leaf, depth=1)
+      @batches[depth] ||= Set.new
+      @batches[depth] << leaf
+      leaf.parents.each do |parent|
+        build_batch(parent, depth+1)
+      end
+    end
+
+    def filter_nodes
+      @filtered = []
+      top_nodes.each { |node| apply_filter(node) }
+      # draw_full_graph option is only used internally by All::Grapher
+      update_parents! unless @options[:draw_full_graph]
+      @options[:draw_full_graph] ? @nodes : @filtered
+    end
+
+    # remove missing parents references since they will be filtered out
+    def update_parents!
+      @filtered.each do |node|
+        new_parents = node.parents & @filtered
+        node.parents = new_parents
+      end
+    end
+
+    def apply_filter(parent, keep=false)
+      keep ||= @options[:stacks].blank?
+      keep ||= @options[:stacks].include?(parent.name)  # apply filter
+      if keep
+        parent.filtered = true
+        @filtered << parent
+      end
+      parent.children.sort_by(&:name).each do |child|
+        apply_filter(child, keep)
+      end
+    end
+
+    def leaves
+      @nodes.select { |n| n.children.empty? }.sort_by(&:name)
+    end
+
+    def top_nodes
+      @nodes.select { |n| n.parents.empty? }.sort_by(&:name)
+    end
+
+    def save_node(node)
+      @nodes << node unless @nodes.detect { |n| n.name == node.name }
+    end
+  end
+end