terraspace 1.0.4 → 1.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f265061c3973d32dcbea50e12b291ee22041dc2e5bc2cf2521907dd4d74b0703
-  data.tar.gz: 97e6064b47fe37f11660fe05512e1750b23f96cd7bc7ea1bef0aa512884922de
+  metadata.gz: c07e843df9e61e62be834feba56f4d5fa058f3dbb130ed7e15edca66ef23e22d
+  data.tar.gz: a7d7c3946e2485ccb369a5d76d413118aed14592750880a48990ca83cdca5bca
 SHA512:
-  metadata.gz: bb0ccae4efe36fae839a755561e9b6829b998c4ec653f7a9a24c8a67a6e9da9ff9e5b81ad1134018edceed75b5859e12cce280819e2792a203da7a65f4997087
-  data.tar.gz: '0269f857a01274f4c9b64b051c724fff030582e2d62b9222da04ae0828e7770160645a4dee01a0a0c791921197d08ce87f420e5fa24678e961e76fb267c9377b'
+  metadata.gz: 026aa5128ce2bea7deb3e5b0fdb6e99c7cf50297c20f5292bbe6896dabb0ba995e62ecd14526ea2ea7e8e1a4be3895e29021620a89b26df49ad34ee0b95ffd1d
+  data.tar.gz: c2df058d2fe9848834fc4e166c1382ccb0e782170fb52f8fd264e4e948a1032916cc9d77662eac1969f4d2101e66b3655e07d944b217e5749581ac0d3ec04916

data/CHANGELOG.md

@@ -3,6 +3,20 @@
 All notable changes to this project will be documented in this file.
 This project *loosely tries* to adhere to [Semantic Versioning](http://semver.org/), even before v1.0.
 
+## [1.1.1] - 2022-02-02
+- [#199](https://github.com/boltops-tools/terraspace/pull/199) build required dependent stacks as part of terraspace up
+
+## [1.1.0] - 2022-01-30
+- [#196](https://github.com/boltops-tools/terraspace/pull/196) terraspace all: build modules in batches and only each specific stack
+- [#197](https://github.com/boltops-tools/terraspace/pull/197) all plan --output and all up --plan options
+- simplify starter config/app.rb
+
+## [1.0.6] - 2022-01-24
+- [#195](https://github.com/boltops-tools/terraspace/pull/195) improve autodetection for plugin expander for backend like remote
+
+## [1.0.5] - 2022-01-23
+- [#194](https://github.com/boltops-tools/terraspace/pull/194) ability to allow and deny envs, regions, and stacks
+
 ## [1.0.4] - 2022-01-21
 - [#193](https://github.com/boltops-tools/terraspace/pull/193) improve all include_stacks and exclude_stacks option
 

data/lib/templates/base/project/.gitignore

@@ -5,6 +5,7 @@ terraform.tfvars
 *.tfstate*
 
 .terraspace-cache
+*.plan
 
 # OS X files
 .history

data/lib/templates/base/project/config/app.rb

@@ -1,4 +1,4 @@
+# Docs: https://terraspace.cloud/docs/config/reference/
 Terraspace.configure do |config|
   config.logger.level = :info
-  config.test_framework = "rspec"
 end

data/lib/terraspace/all/grapher.rb

@@ -3,14 +3,13 @@ require "tty-tree"
 
 module Terraspace::All
   class Grapher < Base
+    include Terraspace::Compiler::DirsConcern
     include Terraspace::Util::Logging
 
     def run
       check_graphviz!
       logger.info "Building graph..."
-      builder = Terraspace::Builder.new(@options.merge(mod: "placeholder", quiet: true, draw_full_graph: draw_full_graph))
-      builder.run
-      graph = builder.graph
+      graph = build_graph
       if @options[:format] == "text"
         text(graph.top_nodes)
       else
@@ -18,6 +17,15 @@ module Terraspace::All
       end
     end
 
+    def build_graph
+      resolver = Terraspace::Dependency::Resolver.new(@options.merge(quiet: true, draw_full_graph: draw_full_graph))
+      resolver.resolve
+      dependencies = Terraspace::Dependency::Registry.data # populated after build_unresolved
+      graph = Terraspace::Dependency::Graph.new(stack_names, dependencies, @options)
+      graph.build
+      graph
+    end
+
     def text(nodes)
       Rainbow.enabled = false unless @options[:full]
       data = build_tree_data(nodes)

data/lib/terraspace/all/runner.rb

@@ -1,6 +1,7 @@
 module Terraspace::All
   class Runner < Base
     include Terraspace::Util
+    extend Memoist
 
     def initialize(command, options={})
       @command, @options = command, options
@@ -21,32 +22,27 @@ module Terraspace::All
     end
 
     def build_batches
-      @batches = run_builder(quiet: false)
+      @batches = Terraspace::Dependency::Resolver.new(@options).resolve
       @batches.reverse! if @command == "down"
       @batches
     end
 
     def deploy_batches
       truncate_logs if ENV['TS_TRUNCATE_LOGS']
+      build_modules
       @batches.each_with_index do |batch,i|
         logger.info "Batch Run #{i+1}:"
-        run_builder unless i == 0 # already handled by build_batches the first time
         deploy_batch(batch)
       end
     end
 
-    # Should run after each batch run. run_builder also calls replace_outputs.
-    # Important: rebuild from source so placeholders are in place.
-    def run_builder(quiet: true)
-      Terraspace::Builder.new(@options.merge(mod: "placeholder", quiet: quiet)).run
-    end
-
     def deploy_batch(batch)
       @pids = {} # stores child processes pids. map of pid to mod_name, reset this list on each batch run
       concurrency = Terraspace.config.all.concurrency
       batch.sort_by(&:name).each_slice(concurrency) do |slice|
         slice.each do |node|
           pid = fork do
+            build_stack(node.name)
             run_terraspace(node.name)
           end
           @pids[pid] = node.name # store mod_name mapping
@@ -57,6 +53,16 @@ module Terraspace::All
       report_errors # reports finall errors and possibly exit
     end
 
+    def build_modules
+      builder = Terraspace::Builder.new(@options.merge(mod: "placeholder", clean: true, quiet: true, include_stacks: :none))
+      builder.build(modules: true)
+    end
+
+    def build_stack(name)
+      builder = Terraspace::Builder.new(@options.merge(mod: name, clean: false, quiet: true, include_stacks: :root_only))
+      builder.build(modules: false)
+    end
+
     def wait_for_child_proccess
       @errors = [] # stores child processes pids that errored
       @pids.each do |pid, _|
@@ -70,7 +76,7 @@ module Terraspace::All
       @errors.each do |pid|
         mod_name = @pids[pid]
         terraspace_command = terraspace_command(mod_name)
-        logger.error "Error running: #{terraspace_command}. Check logs and fix the error.".color(:red)
+        logger.error "Error running: #{terraspace_command}. Fix the error above or check logs for the error.".color(:red)
       end
       unless @errors.empty?
         exit 2 if exit_on_fail?
@@ -97,7 +103,8 @@ module Terraspace::All
           log_path: log_path(mod_name),
           terraspace_command: terraspace_command(mod_name),
         }
-        Summary.new(data).run
+        # Its possible for log file to not get created if RemoteState::Fetcher#validate! fails
+        Summary.new(data).run if File.exist?(data[:log_path])
       end
     end
 

data/lib/terraspace/app/callable_option/concern.rb

@@ -0,0 +1,12 @@
+class Terraspace::App::CallableOption
+  module Concern
+    def callable_option(options={})
+      callable_option = Terraspace::App::CallableOption.new(
+        config_name: options[:config_name],
+        config_value: options[:config_value],
+        passed_args: options[:passed_args],
+      )
+      callable_option.object
+    end
+  end
+end

data/lib/terraspace/app/callable_option.rb

@@ -0,0 +1,58 @@
+# Class represents a terraspace option that is possibly callable. Examples:
+#
+#    config.allow.envs
+#    config.allow.regions
+#    config.deny.envs
+#    config.deny.regions
+#    config.all.include_stacks
+#    config.all.exclude_stacks
+#
+# Abstraction is definitely obtuse. Using it to get rid of duplication.
+#
+class Terraspace::App
+  class CallableOption
+    include Terraspace::Util::Logging
+
+    def initialize(options={})
+      @options = options
+      # Example:
+      # config_name:  config.allow.envs
+      # config_value: ["dev"]
+      # args:         [@stack_name] # passed to object.call
+      @config_name = options[:config_name]
+      @config_value = options[:config_value]
+      @passed_args = options[:passed_args]
+    end
+
+    # Returns either an Array or nil
+    def object
+      case @config_value
+      when nil
+        return nil
+      when Array
+        return @config_value
+      when -> (c) { c.respond_to?(:public_instance_methods) && c.public_instance_methods.include?(:call) }
+        object= @config_value.new
+      when -> (c) { c.respond_to?(:call) }
+        object = @config_value
+      else
+        raise "Invalid option for #{@config_name}"
+      end
+
+      if object
+        result = @passed_args.empty? ? object.call : object.call(*@passed_args)
+        unless result.is_a?(Array) || result.is_a?(NilClass)
+          message = "ERROR: The #{@config_name} needs to return an Array or nil"
+          logger.info message.color(:yellow)
+          logger.info <<~EOL
+            The #{@config_name} when assigned a class, object, or proc must implement
+            the call method and return an Array or nil.
+            The current return value is a #{result.class}
+          EOL
+          raise message
+        end
+      end
+      result
+    end
+  end
+end

data/lib/terraspace/app.rb

@@ -12,17 +12,30 @@ module Terraspace
 
     def defaults
       config = ActiveSupport::OrderedOptions.new
+
       config.all = ActiveSupport::OrderedOptions.new
       config.all.concurrency = 5
       config.all.exit_on_fail = ActiveSupport::OrderedOptions.new
-      config.all.exit_on_fail.down = true
+      config.all.exit_on_fail.down = false
+      config.all.exit_on_fail.plan = true
       config.all.exit_on_fail.up = true
-      config.all.ignore_stacks = nil
-      config.all.include_stacks = nil
+
       config.allow = ActiveSupport::OrderedOptions.new
       config.allow.envs = nil
       config.allow.regions = nil
+      config.allow.stacks = nil
+      config.deny = ActiveSupport::OrderedOptions.new
+      config.deny.envs = nil
+      config.deny.regions = nil
+      config.deny.stacks = nil
+
+      config.all.exclude_stacks = nil
+      config.all.include_stacks = nil
+      config.all.consider_allow_deny_stacks = true
+
       config.auto_create_backend = true
+      config.autodetect = ActiveSupport::OrderedOptions.new
+      config.autodetect.expander = nil
       config.build = ActiveSupport::OrderedOptions.new
       config.build.cache_dir = ":CACHE_ROOT/:REGION/:ENV/:BUILD_DIR"
       config.build.cache_root = nil # defaults to /full/path/to/.terraspace-cache

data/lib/terraspace/autodetect.rb

@@ -2,13 +2,9 @@ module Terraspace
   class Autodetect
     def plugin
       plugins = Terraspace::Plugin.meta.keys
-      if plugins.size == 1
-        plugins.first
-      else
-        precedence = %w[aws azurerm google]
-        precedence.find do |p|
-          plugins.include?(p)
-        end
+      precedence = %w[aws azurerm google]
+      precedence.find do |p|
+        plugins.include?(p)
       end
     end
   end

data/lib/terraspace/builder/allow/base.rb

@@ -0,0 +1,59 @@
+class Terraspace::Builder::Allow
+  class Base
+    include Terraspace::App::CallableOption::Concern
+
+    def initialize(mod)
+      @mod = mod # Only Region subclass uses @mod but keeping interface same for Env for simplicity
+      @stack_name = mod.name
+    end
+
+    def check!
+      messages = []
+      unless allowed?
+        messages << message # message is interface method
+      end
+      unless messages.empty?
+        puts "ERROR: The configs do not allow this.".color(:red)
+        puts messages
+        exit 1
+      end
+    end
+
+    def allowed?
+      if allows.nil? && denys.nil?
+        true
+      elsif denys.nil?
+        allows.include?(check_value)
+      elsif allows.nil?
+        !denys.include?(check_value)
+      else
+        allows.include?(check_value) && !denys.include?(check_value)
+      end
+    end
+
+    def allows
+      callable_option(
+        config_name: "config.allow.#{config_name}",
+        config_value: config.dig(:allow, config_name),
+        passed_args: [@stack_name],
+      )
+    end
+
+    def denys
+      callable_option(
+        config_name: "config.deny.#{config_name}",
+        config_value: config.dig(:deny, config_name),
+        passed_args: [@stack_name],
+      )
+    end
+
+  private
+    def config
+      Terraspace.config
+    end
+
+    def config_name
+      self.class.to_s.split('::').last.underscore.pluralize.to_sym # ActiveSuport::HashWithIndifferentAccess#dig requires symbol
+    end
+  end
+end

data/lib/terraspace/builder/allow/env.rb

@@ -0,0 +1,17 @@
+class Terraspace::Builder::Allow
+  class Env < Base
+    # interface method
+    def message
+      messages = []
+      messages << "This env is not allowed to be used: TS_ENV=#{Terraspace.env}"
+      messages << "Allow envs: #{allows.join(', ')}" if allows
+      messages << "Deny envs: #{denys.join(', ')}" if denys
+      messages.join("\n")
+    end
+
+    # interface method
+    def check_value
+      Terraspace.env
+    end
+  end
+end

data/lib/terraspace/builder/allow/region.rb

@@ -0,0 +1,31 @@
+class Terraspace::Builder::Allow
+  class Region < Base
+    # interface method
+    def message
+      messages = []
+      word = config_name.to_s # IE: regions or locations
+      messages << "This #{word.singularize} is not allowed to be used: Detected current #{word.singularize}=#{current_region}"
+      messages << "Allow #{word}: #{allows.join(', ')}" if allows
+      messages << "Deny #{word}: #{denys.join(', ')}" if denys
+      messages.join("\n")
+    end
+
+    # interface method
+    def check_value
+      current_region
+    end
+
+    def current_region
+      expander = Terraspace::Compiler::Expander.autodetect(@mod).expander
+      expander.region
+    end
+
+    def config_name
+      if config.allow.locations || config.deny.locations
+        :locations # ActiveSuport::HashWithIndifferentAccess#dig requires symbol
+      else
+        super # :regions
+      end
+    end
+  end
+end

data/lib/terraspace/builder/allow/stack.rb

@@ -0,0 +1,17 @@
+class Terraspace::Builder::Allow
+  class Stack < Base
+    # interface method
+    def message
+      messages = []
+      messages << "This stack is not allowed to be used for TS_ENV=#{Terraspace.env}"
+      messages << "Allow stacks: #{allows.join(', ')}" if allows
+      messages << "Deny stacks: #{denys.join(', ')}" if denys
+      messages.join("\n")
+    end
+
+    # interface method
+    def check_value
+      @mod.name
+    end
+  end
+end

data/lib/terraspace/builder/allow.rb

@@ -5,39 +5,9 @@ class Terraspace::Builder
     end
 
     def check!
-      messages = []
-      unless env_allowed?
-        messages << "This env is not allowed to be used: TS_ENV=#{Terraspace.env}"
-        messages << "Allowed envs: #{config.allow.envs.join(', ')}"
-      end
-      unless region_allowed?
-        messages << "This region is not allowed to be used: Detected current region=#{current_region}"
-        messages << "Allowed regions: #{config.allow.regions.join(', ')}"
-      end
-      unless messages.empty?
-        puts "ERROR: The configs do not allow this.".color(:red)
-        puts messages
-        exit 1
-      end
-    end
-
-    def env_allowed?
-      return true unless config.allow.envs
-      config.allow.envs.include?(Terraspace.env)
-    end
-
-    def region_allowed?
-      return true unless config.allow.regions
-      config.allow.regions.include?(current_region)
-    end
-
-    def current_region
-      expander = Terraspace::Compiler::Expander.autodetect(@mod).expander
-      expander.region
-    end
-
-    def config
-      Terraspace.config
+      Env.new(@mod).check!
+      Stack.new(@mod).check!
+      Region.new(@mod).check!
     end
   end
 end

data/lib/terraspace/builder/children.rb

@@ -0,0 +1,42 @@
+class Terraspace::Builder
+  class Children
+    include Terraspace::Util::Logging
+
+    def initialize(mod, options={})
+      @mod, @options = mod, options
+      @queue = []
+    end
+
+    def build
+      dependencies = Terraspace::Dependency::Registry.data
+      # Find out if current deploy stack contains dependency
+      found = dependencies.find do |parent_child|
+        parent, _ = parent_child.split(':')
+        parent == @mod.name
+      end
+      return unless found
+
+      # Go down graph children, which are the dependencies to build a queue
+      parent, _ = found.split(':')
+      node = Terraspace::Dependency::Node.find_by(name: parent)
+      build_queue(node)
+
+      logger.debug "Terraspace::Builder::Children @queue #{@queue}"
+
+      # Process queue in reverse order to build leaf nodes first
+      @queue.reverse.each do |node|
+        mod = Terraspace::Mod.new(node.name, @options)
+        Terraspace::Compiler::Perform.new(mod).compile
+      end
+    end
+
+    # Use depth first traversal to build queue
+    def build_queue(node)
+      node.children.each do |child|
+        @queue << child
+        build_queue(child)
+      end
+      @queue
+    end
+  end
+end

data/lib/terraspace/builder.rb

@@ -4,64 +4,66 @@ module Terraspace
     include Compiler::DirsConcern
     include Hooks::Concern
 
-    attr_reader :graph
+    # @include_stacks can be 3 values: root_with_children, none, root_only
+    #
+    #     none: dont build any stacks at all. used by `terraspace all up`
+    #     root_only: only build root stack. used by `terraspace all up`
+    #     root_with_children: build all parent stacks as well as the root stack. normal `terraspace up`
+    #
+    def initialize(options={})
+      super
+      @include_stacks = @options[:include_stacks] || :root_with_children
+    end
 
     def run
       return if @options[:build] == false
       Terraspace::CLI::Setup::Check.check!
+      check_allow!
       @mod.root_module = true
       clean
+      resolve_dependencies if @include_stacks == :root_with_children
+      build
+    end
+
+    def resolve_dependencies
+      resolver = Terraspace::Dependency::Resolver.new(@options.merge(quiet: true))
+      resolver.resolve # returns batches
+    end
+
+    def build(modules: true)
       build_dir = Util.pretty_path(@mod.cache_dir)
       placeholder_stack_message
       logger.info "Building #{build_dir}" unless @options[:quiet] # from terraspace all
-
-      batches = nil
       FileUtils.mkdir_p(@mod.cache_dir) # so terraspace before build hooks work
       run_hooks("terraspace.rb", "build") do
-        check_allow!
-        build_unresolved
-        batches = build_batches
-        build_all
+        build_dir("modules") if modules
+        build_stacks
         logger.info "Built in #{build_dir}" unless @options[:quiet] # from terraspace all
       end
-      batches
     end
 
     def check_allow!
       Allow.new(@mod).check!
     end
 
-    # Builds dependency graph and returns the batches to run
-    def build_batches
-      dependencies = Terraspace::Dependency::Registry.data # populated after build_unresolved
-      @graph = Terraspace::Dependency::Graph.new(stack_names, dependencies, @options)
-      @graph.build
-    end
-
-    def build_all
-      # At this point dependencies have been resolved.
-      Terraspace::Terraform::RemoteState::Fetcher.flush!
-      @resolved = true
-      build_unresolved
-    end
-
-    def build_unresolved
-      build_dir("modules")
-      build_dir("stacks")
-      build_root_module
+    def build_stacks
+      return if @include_stacks == :none
+      build_children_stacks if @include_stacks == :root_with_children
+      Compiler::Perform.new(@mod).compile # @include_stacks :root or :root_with_children
     end
 
-    def build_root_module
-      @mod.resolved = @resolved
-      Compiler::Builder.new(@mod).build
+    # Build stacks that are part of the dependency graph. Because .terraspace-cache folders
+    # need to exist so `terraform state pull` works to get the state info.
+    def build_children_stacks
+      children = Children.new(@mod, @options)
+      children.build
     end
 
     def build_dir(type_dir)
       with_each_mod(type_dir) do |mod|
-        mod.resolved = @resolved
         is_root_module = mod.cache_dir == @mod.cache_dir
-        next if is_root_module # handled by build_root_module
-        Compiler::Builder.new(mod).build
+        next if is_root_module # handled by build_stacks
+        Compiler::Perform.new(mod).compile
       end
     end
 

data/lib/terraspace/cli/all.rb

@@ -38,6 +38,7 @@ class Terraspace::CLI
 
     desc "plan", "Run plan for all or multiple stacks."
     long_desc Help.text("all/plan")
+    option :out, aliases: :o, desc: "Output path. Can be a pattern like :MOD_NAME.plan"
     def plan(*stacks)
       Terraspace::All::Runner.new("plan", @options.merge(stacks: stacks)).run
     end
@@ -56,6 +57,7 @@ class Terraspace::CLI
 
     desc "up", "Deploy all or multiple stacks."
     long_desc Help.text("all/up")
+    option :plan, desc: "Plan path. Can be a pattern like :MOD_NAME.plan"
     def up(*stacks)
       Terraspace::All::Runner.new("up", @options.merge(stacks: stacks)).run
     end

data/lib/terraspace/cli/build/placeholder.rb

@@ -3,6 +3,7 @@
 # It's useful for the summary command.
 module Terraspace::CLI::Build
   class Placeholder
+    include Terraspace::Compiler::DirsConcern
     include Terraspace::Util::Logging
 
     def initialize(options={})
@@ -28,7 +29,8 @@ module Terraspace::CLI::Build
     def find_stack
       stack_paths = Dir.glob("{app,vendor}/stacks/*")
       stack_paths.select! do |path|
-        select = Terraspace::Compiler::Select.new(path)
+        stack_name = extract_stack_name(path)
+        select = Terraspace::Compiler::Select.new(stack_name)
         select.selected?
       end
       mod_path = stack_paths.last

data/lib/terraspace/cli/help/all/plan.md

@@ -22,4 +22,14 @@
     terraspace plan a1:  Plan: 2 to add, 0 to change, 0 to destroy.
     terraspace plan a1:  Changes to Outputs:
     Time took: 11s
-    $
+    $
+
+## Using Plan Outputs
+
+Using plan output path. You can specify an output path for the plan that contains pattern for expansion. Example:
+
+    $ terraspace all plan --out ":MOD_NAME.plan"
+
+You can then use this later in terraspace up:
+
+    $ terraspace all up --plan ":MOD_NAME.plan"

data/lib/terraspace/cli/help/all/up.md

@@ -25,3 +25,13 @@ Once you confirm, Terraspace deploys the batches in parallel. Essentially, Terra
     Time took: 25s
 
 Terraspace provides a reduced-noise summary of the runs. The full logs are also written for further inspection and debugging. The [terraspace log](https://terraspace.cloud/reference/terraspace-log/) command is useful for viewing the logs.
+
+## Using Plans
+
+Using plan output path. You can specify an output path for the plan that contains pattern for expansion. Example:
+
+    $ terraspace all plan --out ":MOD_NAME.plan"
+
+You can then use this later in terraspace up:
+
+    $ terraspace all up --plan ":MOD_NAME.plan"