terraspace 0.2.3 → 0.3.3

data/lib/terraspace/all/summary.rb

@@ -0,0 +1,99 @@
+module Terraspace::All
+  class Summary
+    include Terraspace::CLI::Log::Concern
+    include Terraspace::Util::Logging
+
+    def initialize(data={})
+      @data = data
+      @command = data[:command]
+      @log_path = data[:log_path]
+      @terraspace_command = data[:terraspace_command]
+    end
+
+    @@header_shown = false
+    def run
+      @lines = readlines(@log_path)
+      if respond_to?(@command.to_sym)
+        send(@command)
+      else
+        default
+      end
+      summarize
+    end
+
+    # Examples of "complete" line:
+    #     [2020-09-06T21:58:25 #11313 terraspace up b1]: Apply complete! Resources: 0 added, 0 changed, 0 destroyed.
+    #     [2020-09-07T08:28:15 #26093 terraspace down a1]: Destroy complete! Resources: 2 destroyed.
+    #
+    # handles: up and down
+    def default
+      @lines.select! do |line|
+        line.include?("complete! Resources:") || # success: handles both apply and destroy output
+        line.include?("Changes to Outputs") ||
+        line.include?("No changes") ||
+        line.include?("Error: ")  # error
+      end
+    end
+
+    # Example 1:
+    #     [2020-09-07T14:45:14 #23340 terraspace plan b1]: No changes. Infrastructure is up-to-date.
+    # Example 2:
+    #     [2020-09-07T14:46:58 #31974 terraspace plan b1]: Changes to Outputs:
+    #     [2020-09-07T14:46:58 #31974 terraspace plan b1]:   ~ length = 1 -> 2
+    def plan
+      @lines.select! do |line|
+        line.include?("No changes. Infrastructure") ||
+        line.include?("Changes to Outputs") ||
+        line.include?("Plan:") ||
+        line.include?("Changes to ") ||
+        line.include?("Error: ")  # error
+      end
+    end
+
+    def output
+      @lines # pass through all output info
+    end
+
+    def show
+      resources = @lines.grep(/: resource "/).count
+      outputs = count_outputs(@lines)
+      summary = "Resources: #{resources} Outputs: #{outputs}"
+      # get summary line before running select! which will remove lines
+      @lines.select! do |line|
+        line.include?("Error: ")  # error
+      end
+      @lines.unshift(summary) # add to top
+    end
+
+    # [2020-09-07T14:53:36 #31106 terraspace show b1]: Outputs:
+    # [2020-09-07T14:53:36 #31106 terraspace show b1]:
+    # [2020-09-07T14:53:36 #31106 terraspace show b1]: length = 1
+    # [2020-09-07T14:53:36 #31106 terraspace show b1]: length2 = 2
+    # [2020-09-07T14:53:36 #31106 terraspace show b1]: random_pet_id = "corgi"
+    def count_outputs(lines)
+      count = 0
+      counting = false
+      lines.each do |line|
+        counting ||= line.include?(": Outputs:")
+        count += 1 if counting && line.include?(' = ')
+      end
+      count
+    end
+
+    # [2020-09-07T13:51:45 #21323 terraspace validate a1]: Success! The configuration is valid.
+    def validate
+      @lines.select! do |line|
+        line.include?("The configuration is") || # success
+        line.include?("Error: ")  # error
+      end
+    end
+
+  private
+    def summarize
+      @lines.each do |line|
+        line.sub!(/.*\]:\s+/, ' ') # remove log info from line
+        logger.info("#{@terraspace_command}: #{line}")
+      end
+    end
+  end
+end

data/lib/terraspace/app.rb

@@ -10,18 +10,40 @@ module Terraspace
 
     def defaults
       config = ActiveSupport::OrderedOptions.new
-      config.test_framework = "rspec"
-      config.logger = Logger.new($stdout)
-      config.logger.level = :info
-      config.hooks = Hooks.new
-      config.cloud = ActiveSupport::OrderedOptions.new
-      config.cloud.overwrite = true
-      config.cloud.overwrite_sensitive = true
-      config.cloud.relative_root = nil
+      config.all = ActiveSupport::OrderedOptions.new
+      config.all.concurrency = 5
+      config.all.exit_on_fail = ActiveSupport::OrderedOptions.new
+      config.all.exit_on_fail.down = true
+      config.all.exit_on_fail.up = true
+      config.all.ignore_stacks = []
+      config.auto_create_backend = true
       config.build = ActiveSupport::OrderedOptions.new
-      config.build.cache_root = nil # defaults to /full/path/to/.terraspace-cache
       config.build.cache_dir = ":CACHE_ROOT/:REGION/:ENV/:BUILD_DIR"
+      config.build.cache_root = nil # defaults to /full/path/to/.terraspace-cache
       config.build.clean_cache = nil # defaults to /full/path/to/.terraspace-cache
+      config.cloud = ActiveSupport::OrderedOptions.new
+      config.cloud.auto_sync = true
+      config.cloud.working_dir_prefix = nil
+      config.cloud.hostname = nil
+      config.cloud.vars = ActiveSupport::OrderedOptions.new
+      config.cloud.vars.overwrite = true
+      config.cloud.vars.overwrite_sensitive = true
+      config.cloud.vars.show_message = "create"
+      config.cloud.workspace = ActiveSupport::OrderedOptions.new
+      config.cloud.workspace.attrs = ActiveSupport::OrderedOptions.new
+      config.hooks = Hooks.new
+      config.init = ActiveSupport::OrderedOptions.new
+      config.init.mode = "auto" # auto, never, always
+      config.log = ActiveSupport::OrderedOptions.new
+      config.log.root = Terraspace.log_root
+      config.logger = Logger.new(ENV['TS_LOG_PATH'] || $stderr)
+      config.logger.level = ENV['TS_LOG_LEVEL'] || :info
+      config.logger.formatter = Logger::Formatter.new
+      config.test_framework = "rspec"
+      config.terraform = ActiveSupport::OrderedOptions.new
+      config.terraform.plugin_cache = ActiveSupport::OrderedOptions.new
+      config.terraform.plugin_cache.enabled = true
+      config.terraform.plugin_cache.dir = ENV['TF_PLUGIN_CACHE_DIR'] || "#{Terraspace.tmp_root}/plugin_cache"
       config
     end
 

data/lib/terraspace/booter.rb

@@ -5,6 +5,7 @@ module Terraspace
       load_plugin_default_configs
       Terraspace.config # load project config
       Terraspace::App::Hooks.run_hook(:on_boot)
+      set_plugin_cache!
     end
 
     def load_plugin_default_configs
@@ -14,6 +15,14 @@ module Terraspace
       end
     end
 
+    def set_plugin_cache!
+      plugin_cache = Terraspace.config.terraform.plugin_cache
+      return unless plugin_cache.enabled
+      dir = ENV['TF_PLUGIN_CACHE_DIR'] ||= plugin_cache.dir
+      FileUtils.mkdir_p(dir)
+      dir
+    end
+
     extend self
   end
 end

data/lib/terraspace/builder.rb

@@ -1,48 +1,85 @@
 module Terraspace
   class Builder < Terraspace::CLI::Base
+    include Compiler::DirsConcern
+    include Compiler::CommandsConcern
+
+    attr_reader :graph
+
     def run
+      return if @options[:build] == false
       Terraspace::CLI::CheckSetup.check!
       @mod.root_module = true
-      Compiler::Cleaner.new(@mod, @options).clean if clean?
+      clean
       build_dir = Util.pretty_path(@mod.cache_dir)
-      logger.info "Building #{build_dir}"
+      placeholder_stack_message
+      logger.info "Building #{build_dir}" unless @options[:quiet] # from terraspace all
+
+      build_unresolved
+      auto_create_backend
+      batches = build_batches
+      build_all
+      logger.info "Built in #{build_dir}" unless @options[:quiet] # from terraspace all
+      batches
+    end
+
+    # Builds dependency graph and returns the batches to run
+    def build_batches
+      dependencies = Terraspace::Dependency::Registry.data # populated after build_unresolved
+      @graph = Terraspace::Dependency::Graph.new(stack_names, dependencies, @options)
+      @graph.build
+    end
 
-      build_all("modules") # build all modules and stacks as dependencies
-      build_all("stacks")
+    def build_all
+      # At this point dependencies have been resolved.
+      Terraspace::Terraform::RemoteState::Fetcher.flush!
+      @resolved = true
+      build_unresolved
+    end
+
+    def build_unresolved
+      build_dir("modules")
+      build_dir("stacks")
       build_root_module
-      logger.info "Built in #{build_dir}"
     end
 
     def build_root_module
+      @mod.resolved = @resolved
       Compiler::Builder.new(@mod).build
     end
 
-    def build_all(type_dir)
-      built = []
-      local_paths(type_dir).each do |path|
-        next unless File.directory?(path)
-        mod_name = File.basename(path)
-        next if built.include?(mod_name) # ensures modules in app folder take higher precedence than vendor folder
-
-        consider_stacks = type_dir == "stacks"
-        mod = Mod.new(mod_name, consider_stacks: consider_stacks)
-
+    def build_dir(type_dir)
+      with_each_mod(type_dir) do |mod|
+        mod.resolved = @resolved
+        is_root_module = mod.cache_dir == @mod.cache_dir
+        next if is_root_module # handled by build_root_module
         Compiler::Builder.new(mod).build
-        built << mod_name
       end
     end
 
-    def local_paths(type_dir)
-      dirs("app/#{type_dir}/*") + dirs("vendor/#{type_dir}/*")
+    # Auto create after build_unresolved since will need to run state pull for dependencies
+    def auto_create_backend
+      return if Terraspace.config.auto_create_backend == false
+      return unless requires_backend?
+      Terraspace::Compiler::Backend.new(@mod).create
     end
 
-    def dirs(path)
-      Dir.glob("#{Terraspace.root}/#{path}")
+    def clean
+      Compiler::Cleaner.new(@mod, @options).clean if clean?
     end
 
     def clean?
-      clean_cache = Terraspace.config.build.clean_cache
-      clean_cache.nil? ? true : clean_cache
+      if @options[:clean].nil?
+        clean_cache = Terraspace.config.build.clean_cache
+        clean_cache.nil? ? true : clean_cache
+      else
+        @options[:clean]
+      end
+    end
+
+    def placeholder_stack_message
+      return if @options[:quiet]
+      return unless @options[:mod] == "placeholder"
+      logger.info "Building one stack to build all stacks"
     end
   end
 end

data/lib/terraspace/cli.rb

@@ -28,21 +28,29 @@ module Terraspace
       option :reconfigure, type: :boolean, desc: "Add terraform -reconfigure option"
     }
 
-    desc "new SUBCOMMAND", "new subcommands"
-    long_desc Help.text(:new)
-    subcommand "new", New
+    desc "all SUBCOMMAND", "all subcommands"
+    long_desc Help.text(:all)
+    subcommand "all", All
 
     desc "cloud SUBCOMMAND", "cloud subcommands"
     long_desc Help.text(:cloud)
     subcommand "cloud", Cloud
 
-    desc "build STACK", "build"
+    desc "logs SUBCOMMAND", "logs management subcommands"
+    long_desc Help.text(:logs)
+    subcommand "logs", Logs
+
+    desc "new SUBCOMMAND", "new subcommands"
+    long_desc Help.text(:new)
+    subcommand "new", New
+
+    desc "build [STACK]", "build"
     long_desc Help.text(:build)
-    option :quiet, type: :boolean, default: true, desc: "quiet output"
+    option :quiet, type: :boolean, desc: "quiet output"
     instance_option.call
     yes_option.call
-    def build(mod)
-      Terraspace::Builder.new(@options.merge(mod: mod)).run
+    def build(mod="placeholder")
+      Terraspace::Builder.new(options.merge(mod: mod)).run # building any stack builds them all
     end
 
     desc "bundle", "bundle"
@@ -77,8 +85,7 @@ module Terraspace
     reconfigure_option.call
     option :destroy_workspace, type: :boolean, desc: "Also destroy the Cloud workspace. Only applies when using Terraform Cloud remote backend."
     def down(mod)
-      Commander.new("destroy", options.merge(mod: mod, command: "down")).run
-      Terraspace::Terraform::Cloud::Workspace.new(options.merge(mod: mod)).destroy if @options[:destroy_workspace]
+      Down.new(options.merge(mod: mod)).run
     end
 
     desc "info STACK", "info"
@@ -89,12 +96,30 @@ module Terraspace
       Info.new(options.merge(mod: mod)).run
     end
 
+    desc "init STACK", "init"
+    long_desc Help.text(:init)
+    instance_option.call
+    def init(mod)
+      Commander.new("init", options.merge(mod: mod, quiet: false)).run
+    end
+
     desc "list", "list stacks and modules"
     long_desc Help.text(:list)
+    option :type, aliases: %w[t], desc: "Type: stack or module. Default all"
     def list
       List.new(options).run
     end
 
+    desc "log [ACTION] [STACK]", "The log command allows you to view multiple logs."
+    long_desc Help.text("log")
+    option :timestamps, aliases: %w[t], type: :boolean, desc: "Whether or not to show the leading timestamp. Defaults to timestamps for multiple logs, and no timestamp if a single log is specified. Note: In follow mode, timestamp always shown"
+    option :follow, aliases: %w[f], type: :boolean, desc: "Follow the log in live tail fashion. Must specify a stack if using this option."
+    option :limit, aliases: %w[n], default: 10, type: :numeric, desc: "Number of lines to limit showing. Only applies in no-follow mode."
+    option :all, aliases: %w[a], type: :boolean, desc: "All mode turns off the limit. Defaults to all if a single log is specified. Only applies in no-follow mode."
+    def log(action=nil, stack=nil)
+      Log.new(@options.merge(action: action, stack: stack)).run
+    end
+
     desc "plan STACK", "plan stack"
     long_desc Help.text(:plan)
     auto_option.call
@@ -142,6 +167,8 @@ module Terraspace
     desc "show STACK", "show"
     long_desc Help.text(:show)
     instance_option.call
+    option :plan, desc: "path to created.plan"
+    option :json, type: :boolean, desc: "show plan in json format"
     def show(mod)
       Commander.new("show", options.merge(mod: mod)).run
     end
@@ -161,7 +188,7 @@ module Terraspace
       Commander.new("output", options.merge(mod: mod)).run
     end
 
-    desc "update STACK", "Update infrasturcture. IE: apply plan"
+    desc "up STACK", "Deploy infrastructure. IE: terraform apply"
     long_desc Help.text(:update)
     auto_option.call
     init_option.call
@@ -171,8 +198,8 @@ module Terraspace
     reconfigure_option.call
     option :plan, desc: "Execution plan that can be used to only execute a pre-determined set of actions."
     option :var_files, type: :array, desc: "list of var files"
-    def update(mod)
-      Commander.new("apply", options.merge(mod: mod)).run
+    def up(mod)
+      Up.new(options.merge(mod: mod)).run
     end
 
     desc "validate STACK", "validate"

data/lib/terraspace/cli/all.rb

@@ -0,0 +1,63 @@
+class Terraspace::CLI
+  class All < Terraspace::Command
+    class_option :yes, aliases: :y, type: :boolean, desc: "auto approve all batch commands"
+    class_option :exit_on_fail, type: :boolean, desc: "whether or not to exit when one of the batch commands fails"
+
+    desc "down", "destroy all"
+    long_desc Help.text("all/down")
+    option :destroy_workspace, type: :boolean, desc: "Also destroy the Cloud workspace. Only applies when using Terraform Cloud remote backend."
+    def down(*stacks)
+      Terraspace::All::Runner.new("down", @options.merge(stacks: stacks)).run
+    end
+
+    desc "graph", "graph all"
+    long_desc Help.text("all/graph")
+    option :format, default: "png", desc: "format: text or diagram"
+    option :full,type: :boolean, desc: "draw the full graph with highlighted nodes or draw the subgraph. text format defaults to false, graph format defaults to true"
+    def graph(*stacks)
+      Terraspace::All::Grapher.new(@options.merge(stacks: stacks)).run
+    end
+
+    desc "refresh", "refresh all"
+    long_desc Help.text("all/refresh")
+    def refresh(*stacks)
+      Terraspace::All::Runner.new("refresh", @options.merge(stacks: stacks)).run
+    end
+
+    desc "output", "output all"
+    long_desc Help.text("all/output")
+    def output(*stacks)
+      Terraspace::All::Runner.new("output", @options.merge(stacks: stacks)).run
+    end
+
+    desc "plan", "plan all"
+    long_desc Help.text("all/plan")
+    def plan(*stacks)
+      Terraspace::All::Runner.new("plan", @options.merge(stacks: stacks)).run
+    end
+
+    desc "providers", "providers all"
+    long_desc Help.text("all/providers")
+    def providers(*stacks)
+      Terraspace::All::Runner.new("providers", @options.merge(stacks: stacks)).run
+    end
+
+    desc "show", "show all"
+    long_desc Help.text("all/show")
+    def show(*stacks)
+      Terraspace::All::Runner.new("show", @options.merge(stacks: stacks)).run
+    end
+
+    desc "up", "deploy all"
+    long_desc Help.text("all/up")
+    def up(*stacks)
+      Terraspace::All::Runner.new("up", @options.merge(stacks: stacks)).run
+    end
+
+    desc "validate", "validate all"
+    long_desc Help.text("all/validate")
+    def validate(*stacks)
+      Terraspace::All::Runner.new("validate", @options.merge(stacks: stacks)).run
+    end
+  end
+end