terraspace 1.0.4 → 1.1.1

data/lib/terraspace/cli/help/plan.md

@@ -26,4 +26,12 @@
     can't guarantee that exactly these actions will be performed if
     "terraform apply" is subsequently run.
 
-    $
+    $
+
+Using plan output path. You can specify an output path for the plan. Example:
+
+    $ terraspace plan demo --out "my.plan"
+
+You can then use this later in terraspace up:
+
+    $ terraspace up demo --plan "my.plan"

data/lib/terraspace/cli/help/up.md

@@ -27,4 +27,12 @@
 
     bucket_name = bucket-trusty-marmoset
     Time took: 39s
-    $
+    $
+
+Using plan output path. You can specify an output path for the plan. Example:
+
+    $ terraspace plan demo --out "my.plan"
+
+You can then use this later in terraspace up:
+
+    $ terraspace up demo --plan "my.plan"

data/lib/terraspace/cli/init.rb

@@ -52,7 +52,7 @@ class Terraspace::CLI
       return if meta['Dir'] == '.' # root is already built
 
       remote_mod = Terraspace::Mod::Remote.new(meta, @mod)
-      Terraspace::Compiler::Builder.new(remote_mod).build
+      Terraspace::Compiler::Perform.new(remote_mod).build
     end
 
     def auto?

data/lib/terraspace/cli.rb

@@ -9,7 +9,7 @@ module Terraspace
       option :yes, aliases: :y, type: :boolean, desc: "-auto-approve the terraform apply"
     }
     out_option = Proc.new {
-      option :out, aliases: :o, desc: "Write the output to path"
+      option :out, aliases: :o, desc: "Output path. Can be a pattern like :MOD_NAME.plan"
     }
     input_option = Proc.new {
       option :input, type: :boolean, desc: "Ask for input for variables if not directly set."
@@ -55,6 +55,7 @@ module Terraspace
     option :quiet, type: :boolean, desc: "quiet output"
     instance_option.call
     yes_option.call
+    option :clean, type: :boolean, default: nil, desc: "Whether or not clean out .terraspace-cache folder first", hide: true
     def build(mod="placeholder")
       Terraspace::Builder.new(options.merge(mod: mod)).run # building any stack builds them all
     end
@@ -190,7 +191,7 @@ module Terraspace
     desc "show STACK", "Run show."
     long_desc Help.text(:show)
     instance_option.call
-    option :plan, desc: "path to created.plan"
+    option :plan, desc: "Plan path. Can be a pattern like :MOD_NAME.plan"
     def show(mod, *args)
       Commander.new("show", options.merge(mod: mod, args: args)).run
     end

data/lib/terraspace/compiler/dirs_concern.rb

@@ -37,10 +37,15 @@ module Terraspace::Compiler
     #   path     /home/ec2-user/environment/downloads/infra/app/stacks/demo
     def select_stack?(type_dir, path)
       return true unless type_dir == "stacks"
-      select = Terraspace::Compiler::Select.new(path)
+      stack_name = extract_stack_name(path)
+      select = Terraspace::Compiler::Select.new(stack_name)
       select.selected?
     end
 
+    def extract_stack_name(path)
+      path.sub(%r{.*(app|vendor)/stacks/}, '')
+    end
+
     def local_paths(type_dir)
       dirs("app/#{type_dir}/*") + dirs("vendor/#{type_dir}/*")
     end

data/lib/terraspace/compiler/expander/backend.rb

@@ -12,7 +12,7 @@ class Terraspace::Compiler::Expander
       @mod = mod
     end
 
-    COMMENT = /^\s+#/
+    COMMENT = /^\s*#/
     # Works for both backend.rb DSL and backend.tf ERB
     def detect
       return nil unless src_path # no backend file. returning nil means a local backend

data/lib/terraspace/compiler/expander.rb

@@ -3,8 +3,8 @@ module Terraspace::Compiler
     extend Memoist
     delegate :expand, :expansion, to: :expander
 
-    def initialize(mod, name)
-      @mod, @name = mod, name
+    def initialize(mod, backend=nil)
+      @mod, @backend = mod, backend
     end
 
     def expander
@@ -12,27 +12,51 @@ module Terraspace::Compiler
     end
     memoize :expander
 
+    # IE: TerraspacePluginAws::Interfaces::Expander
     def expander_class
-      # IE: TerraspacePluginAws::Interfaces::Expander
-      klass_name = Terraspace::Plugin.klass("Expander", backend: @name)
-      klass_name ? klass_name.constantize : Terraspace::Plugin::Expander::Generic
-    rescue NameError
+      class_name = expander_class_name
+      class_name ? class_name.constantize : Terraspace::Plugin::Expander::Generic
+    rescue NameError => e
+      logger.debug "#{e.class}: #{e.message}"
       Terraspace::Plugin::Expander::Generic
     end
 
+    def expander_class_name
+      plugin = Terraspace.config.autodetect.expander # contains plugin name. IE: aws, azurerm, google
+      if plugin
+        # early return for user override of autodetection
+        return Terraspace::Plugin.klass("Expander", plugin: plugin) # can return nil
+      end
+
+      backend = @backend || parse_backend
+      class_name = Terraspace::Plugin.klass("Expander", backend: backend) # can return nil
+      unless class_name
+        backend = plugin_backend
+        class_name = Terraspace::Plugin.klass("Expander", backend: backend) # can return nil
+      end
+      class_name
+    end
+
+    # autodetect by parsing backend.tf or backend.rb
+    def parse_backend
+      Backend.new(@mod).detect
+    end
+
+    # autodetect by looking up loaded plugins
+    def plugin_backend
+      plugin = Terraspace::Autodetect.new.plugin # IE: aws, azurerm, google
+      if plugin
+        data = Terraspace::Plugin.meta[plugin]
+        data[:backend] # IE: s3, azurerm, gcs
+      end
+    end
+
     class << self
       extend Memoist
-
       def autodetect(mod, opts={})
-        backend = opts[:backend] || find_backend(mod)
-        new(mod, backend)
+        new(mod, opts)
       end
       memoize :autodetect
-
-      def find_backend(mod)
-        Backend.new(mod).detect
-      end
-      memoize :find_backend
     end
   end
 end

data/lib/terraspace/compiler/{builder → perform}/skip.rb

@@ -1,4 +1,4 @@
-class Terraspace::Compiler::Builder
+class Terraspace::Compiler::Perform
   class Skip
     def initialize(mod, src_path)
       @mod, @src_path = mod, src_path

data/lib/terraspace/compiler/{builder.rb → perform.rb}

@@ -1,5 +1,5 @@
 module Terraspace::Compiler
-  class Builder
+  class Perform
     include CommandsConcern
     include Basename
 
@@ -7,44 +7,45 @@ module Terraspace::Compiler
       @mod = mod
     end
 
-    def build
-      build_config
-      build_module if @mod.resolved
-      build_tfvars unless command_is?(:seed) #  avoid dependencies being built and erroring when backend bucket doesnt exist
+    def compile
+      compile_config
+      compile_module if @mod.resolved
+      compile_tfvars
     end
 
-    # build common config files: provider and backend for the root module
-    def build_config
-      return unless build?
-      build_config_terraform
+    # compile common config files: provider and backend for the root module
+    def compile_config
+      return unless compile?
+      compile_config_terraform
     end
 
-    def build_module
+    def compile_module
       with_mod_file do |src_path|
-        build_mod_file(src_path)
+        compile_mod_file(src_path)
       end
     end
 
-    def build_tfvars
-      return unless build?
-      Strategy::Tfvar.new(@mod).run # writer within Strategy to control file ordering
+    def compile_tfvars(write: true)
+      return unless compile?
+      return if command_is?(:seed) # avoid dependencies being built and erroring when backend bucket doesnt exist
+      Strategy::Tfvar.new(@mod).run(write: write) # writer within Strategy to control file ordering
     end
 
   private
-    def build?
+    def compile?
       @mod.type == "stack" || @mod.root_module?
     end
 
-    def build_config_terraform
+    def compile_config_terraform
       expr = "#{Terraspace.root}/config/terraform/**/*"
       search(expr).each do |path|
         next unless File.file?(path)
         next if path.include?('config/terraform/tfvars')
-        build_config_file(basename(path))
+        compile_config_file(basename(path))
       end
     end
 
-    def build_config_file(file)
+    def compile_config_file(file)
       existing = search("#{@mod.root}/#{file}").first
       return if existing && existing.ends_with?(".tf") # do not overwrite existing backend.tf, provider.tf, etc
 
@@ -52,10 +53,10 @@ module Terraspace::Compiler
         src_path = search("#{@mod.root}/#{basename(file)}").first # existing source. IE: backend.rb in module folder
       end
       src_path ||= search("#{Terraspace.root}/config/terraform/#{file}").first
-      build_mod_file(src_path) if src_path
+      compile_mod_file(src_path) if src_path
     end
 
-    def build_mod_file(src_path)
+    def compile_mod_file(src_path)
       content = Strategy::Mod.new(@mod, src_path).run
       Writer.new(@mod, src_path: src_path).write(content)
     end

data/lib/terraspace/compiler/select.rb

@@ -1,10 +1,10 @@
 module Terraspace::Compiler
   class Select
+    include Terraspace::App::CallableOption::Concern
     include Terraspace::Util::Logging
 
-    def initialize(path)
-      @path = path
-      @stack_name = extract_stack_name(path)
+    def initialize(stack_name)
+      @stack_name = stack_name
     end
 
     def selected?
@@ -22,57 +22,47 @@ module Terraspace::Compiler
     end
 
     def include_stacks
-      include_option(:include_stacks)
+      if config.all.include_stacks
+        config_name = "config.all.include_stacks"
+        config_value = config.dig(:all, :include_stacks)
+      elsif config.all.consider_allow_deny_stacks
+        config_name = "config.allow.stacks"
+        config_value = config.dig(:allow, :stacks)
+      else
+        return
+      end
+      callable_option(
+        config_name: config_name,
+        config_value: config_value,
+        passed_args: [@stack_name],
+      )
     end
 
     def exclude_stacks
-      include_option(:exclude_stacks)
-    end
-
-    def include_option(name)
-      option = all[name]  # IE: include_stacks or exclude_stacks
-      option ||= all[:ignore_stacks] if name == :exclude_stacks
-      case option
-      when nil
-        return nil
-      when Array
-        return option
-      when -> (c) { c.respond_to?(:public_instance_methods) && c.public_instance_methods.include?(:call) }
-        object= option.new
-      when -> (c) { c.respond_to?(:call) }
-        object = option
+      if config.all.exclude_stacks
+        config_name = "config.all.exclude_stacks"
+        config_value = config.dig(:all, :exclude_stacks)
+      elsif config.all.consider_allow_deny_stacks
+        config_name = "config.deny.stacks"
+        config_value = config.dig(:deny, :stacks)
       else
-        raise "Invalid option for config.all.#{name}"
+        return
       end
-
-      if object
-        result = object.call(@stack_name)
-        unless result.is_a?(Array) || result.is_a?(NilClass)
-          message = "ERROR: The config.all.#{name} needs to return an Array or nil"
-          logger.info message.color(:yellow)
-          logger.info <<~EOL
-            The config.all.#{name} when assigned a class, object, or proc must implement
-            the call method and return an Array or nil.
-            The current return value is a #{result.class}
-          EOL
-          raise message
-        end
-      end
-      result
+      callable_option(
+        config_name: config_name,
+        config_value: config_value,
+        passed_args: [@stack_name],
+      )
     end
 
   private
-    def all
-      Terraspace.config.all
-    end
-
-    def extract_stack_name(path)
-      path.sub(%r{.*(app|vendor)/stacks/}, '')
+    def config
+      Terraspace.config
     end
 
     @@ignore_stacks_deprecation_warning = nil
     def ignore_stacks_deprecation_warning
-      return unless all.ignore_stacks
+      return unless config.all.ignore_stacks
       return if @@ignore_stacks_deprecation_warning
       puts <<~EOL.color(:yellow)
         DEPRECATED:  config.all.ignore_stacks

data/lib/terraspace/compiler/strategy/tfvar.rb

@@ -5,7 +5,7 @@ module Terraspace::Compiler::Strategy
       @order = 0
     end
 
-    def run
+    def run(write: true)
       layer_paths.each do |layer_path|
         ext = File.extname(layer_path).sub('.','')
         klass = strategy_class(ext)
@@ -14,6 +14,7 @@ module Terraspace::Compiler::Strategy
         strategy = klass.new(@mod, layer_path)
         content = strategy.run
 
+        next unless write
         dest_name = ordered_name(layer_path)
         writer = Terraspace::Compiler::Writer.new(@mod, dest_name: dest_name)
         writer.write(content)

data/lib/terraspace/dependency/resolver.rb

@@ -0,0 +1,20 @@
+module Terraspace::Dependency
+  class Resolver
+    include Terraspace::Compiler::DirsConcern
+
+    def initialize(options={})
+      @options = options
+    end
+
+    def resolve
+      with_each_mod("stacks") do |mod|
+        mod.resolved = false
+        Terraspace::Compiler::Perform.new(mod).compile_tfvars(write: false)
+      end
+
+      dependencies = Terraspace::Dependency::Registry.data # populated dependencies resolved
+      @graph = Terraspace::Dependency::Graph.new(stack_names, dependencies, @options)
+      @graph.build # Returns batches to run
+    end
+  end
+end

data/lib/terraspace/mod.rb

@@ -15,6 +15,7 @@ module Terraspace
       @name, @options = placeholder(name), options
       @consider_stacks = options[:consider_stacks].nil? ? true : options[:consider_stacks]
       @instance = options[:instance]
+      @resolved = true # more common case
     end
 
     def placeholder(name)

data/lib/terraspace/plugin/summary/interface.rb

@@ -49,7 +49,7 @@ module Terraspace::Plugin::Summary
     def download_statefiles
       return unless download?
       FileUtils.rm_rf(@dest_folder)
-      logger.info("Downloading statefiles to #{@dest_folder}")
+      logger.debug("Downloading statefiles to #{@dest_folder}")
       download # INTERFACE METHOD
     end
 

data/lib/terraspace/terraform/args/expand.rb

@@ -0,0 +1,25 @@
+module Terraspace::Terraform::Args
+  class Expand
+    class << self
+      def option_method(*names)
+        names.compact.each do |name|
+          option_method_each(name)
+        end
+      end
+
+      def option_method_each(name)
+        define_method name do
+          return unless @options[name]
+          expander = Terraspace::Compiler::Expander.autodetect(@mod)
+          expander.expansion(@options[name]) # pattern is a String that contains placeholders for substitutions
+        end
+      end
+    end
+
+    def initialize(mod, options={})
+      @mod, @options = mod, options
+    end
+
+    option_method :plan, :out
+  end
+end

data/lib/terraspace/terraform/args/thor.rb

@@ -2,6 +2,8 @@ require "tempfile"
 
 module Terraspace::Terraform::Args
   class Thor
+    extend Memoist
+
     def initialize(mod, name, options={})
       @mod, @name, @options = mod, name.underscore, options
       @quiet = @options[:quiet].nil? ? true : @options[:quiet]
@@ -48,7 +50,7 @@ module Terraspace::Terraform::Args
       args << input_option
 
       # must be at the end
-      plan = @options[:plan]
+      plan = expand.plan
       if plan
         copy_to_cache(plan)
         args << " #{plan}"
@@ -81,7 +83,7 @@ module Terraspace::Terraform::Args
 
     def output_args
       args = []
-      args << "> #{expanded_out}" if @options[:out]
+      args << "> #{expand.out}" if expand.out
       args
     end
 
@@ -89,26 +91,22 @@ module Terraspace::Terraform::Args
       args = []
       args << input_option
       args << "-destroy" if @options[:destroy]
-      args << "-out #{expanded_out}" if @options[:out]
-      # Note: based on the @options[:out] will run an internal hook to copy plan
+      args << "-out #{expand.out}" if expand.out
+      # Note: based on the expand.out will run an internal hook to copy plan
       # file back up to the root project folder for use. Think this is convenient and expected behavior.
       args
     end
 
     def show_args
       args = []
-      plan = @options[:plan]
+      plan = expand.plan
       if plan
-        copy_to_cache(@options[:plan])
-        args << " #{@options[:plan]}" # terraform show /path/to/plan
+        copy_to_cache(expand.plan)
+        args << " #{expand.plan}" # terraform show /path/to/plan
       end
       args
     end
 
-    def expanded_out
-      @options[:out]
-    end
-
     def destroy_args
       auto_approve_arg
     end
@@ -147,5 +145,10 @@ module Terraspace::Terraform::Args
       FileUtils.mkdir_p(File.dirname(dest))
       FileUtils.cp(src, dest) unless same_file?(src, dest)
     end
+
+    def expand
+      Terraspace::Terraform::Args::Expand.new(@mod, @options)
+    end
+    memoize :expand
   end
 end

data/lib/terraspace/terraform/ihooks/after/plan.rb

@@ -1,8 +1,8 @@
 module Terraspace::Terraform::Ihooks::After
   class Plan < Terraspace::Terraform::Ihooks::Base
     def run
-      return if !@options[:out] || @options[:copy_to_root] == false
-      copy_to_root(@options[:out])
+      return if !out_option || @options[:copy_to_root] == false
+      copy_to_root(out_option)
     end
 
     def copy_to_root(file)

data/lib/terraspace/terraform/ihooks/base.rb

@@ -4,5 +4,10 @@ module Terraspace::Terraform::Ihooks
       @name = name
       super(options)
     end
+
+    def out_option
+      expand = Terraspace::Terraform::Args::Expand.new(@mod, @options)
+      expand.out
+    end
   end
 end

data/lib/terraspace/terraform/ihooks/before/plan.rb

@@ -1,12 +1,10 @@
 module Terraspace::Terraform::Ihooks::Before
   class Plan < Terraspace::Terraform::Ihooks::Base
     def run
-      out = @options[:out]
-      return unless out
-      return if out =~ %r{^/} # not need to create parent dir for copy with absolute path
+      return unless out_option
+      return if out_option =~ %r{^/} # not need to create parent dir for copy with absolute path
 
-      out = @options[:out]
-      name = out.sub("#{Terraspace.root}/",'')
+      name = out_option.sub("#{Terraspace.root}/",'')
       dest = "#{@mod.cache_dir}/#{name}"
       FileUtils.mkdir_p(File.dirname(dest))
     end

data/lib/terraspace/terraform/remote_state/fetcher.rb

@@ -2,6 +2,7 @@ module Terraspace::Terraform::RemoteState
   class Fetcher
     extend Memoist
     include Terraspace::Compiler::CommandsConcern
+    include Terraspace::Compiler::DirsConcern
     include Terraspace::Util::Logging
 
     def initialize(parent, identifier, options={})
@@ -47,7 +48,7 @@ module Terraspace::Terraform::RemoteState
     def output_error(type)
       msg = case type
       when :key_not_found
-        "Output #{@output_key} was not found for the #{@parent.name} tfvars file. Either #{@child.name} stack has not been deployed yet or it does not have this output: #{@output_key}"
+        "Output #{@output_key} was not found for the #{@parent.name} tfvars file. Either #{@child.name} stack has not been deployed yet or it does not have this output: #{@output_key}. Also, if local backend is being used and has been removed/cleaned, then it will also result zero-byte state.json with the 'terraform state pull' used to download the terraform state and output will not be found."
       when :state_not_found
         "Output #{@output_key} could not be looked up for the #{@parent.name} tfvars file. #{@child.name} stack needs to be deployed"
       when :bucket_not_found
@@ -62,7 +63,7 @@ module Terraspace::Terraform::RemoteState
     @@download_shown = false
     def pull
       return if @@pull_successes[cache_key]
-      logger.info "Downloading tfstate files for dependencies defined in tfvars..." unless @@download_shown || @options[:quiet]
+      logger.debug "Downloading tfstate files for dependencies defined in tfvars..." unless @@download_shown || @options[:quiet]
       @@download_shown = true
       logger.debug "Downloading tfstate for stack: #{@child.name}"
 
@@ -130,9 +131,15 @@ module Terraspace::Terraform::RemoteState
 
     # Note we already validate mod exist at the terraform_output helper. This is just in case that logic changes.
     def validate!
-      return if @child.exist?
-      logger.error "ERROR: stack #{@child.name} not found".color(:red)
-      exit 1
+      unless @child.exist?
+        logger.error "ERROR: stack #{@child.name} not found".color(:red)
+        exit 1
+      end
+      select = Terraspace::Compiler::Select.new(@child.name)
+      unless select.selected?
+        logger.error "ERROR: stack #{@child.name} is configured to not be included. IE: config.all.include_stacks or config.all.exclude_stacks".color(:red)
+        exit 1
+      end
     end
 
     # Using debug level because all the tfvar files always get evaluated.

data/lib/terraspace/terraform/remote_state/marker/output.rb

@@ -25,7 +25,7 @@ module Terraspace::Terraform::RemoteState::Marker
     end
 
     def warning
-      logger.warn "WARN: The #{@child_name} stack does not exist".color(:yellow)
+      logger.warn "WARN: The #{@child_name} stack does not exist or is configured to be not included. IE: config.all.include_stacks or config.all.exclude_stacks".color(:yellow)
       caller_line = caller.find { |l| l.include?('.tfvars') }
       return unless caller_line # specs dont have a tfvars file
       source_code = PrettyTracer.new(caller_line).source_code

data/lib/terraspace/version.rb

@@ -1,3 +1,3 @@
 module Terraspace
-  VERSION = "1.0.4"
+  VERSION = "1.1.1"
 end

data/spec/terraspace/all/runner_spec.rb

@@ -1,6 +1,7 @@
 describe Terraspace::All::Runner do
   let(:runner) do
     runner = described_class.new(command: "up", yes: true)
+    allow(runner).to receive(:build_modules)
     allow(runner).to receive(:build_batches).and_return(batches)
     allow(runner).to receive(:preview)
     # Just test to the point of the run_builder and deploy_batch

data/spec/terraspace/compiler/erb/render_spec.rb

@@ -1,6 +1,10 @@
 describe Terraspace::Compiler::Erb::Render do
   let(:render) { described_class.new(mod, src_path) }
-  let(:mod)    { Terraspace::Mod.new("a1") }
+  let(:mod)    do
+    mod = Terraspace::Mod.new("a1")
+    mod.resolved = false
+    mod
+  end
 
   # Only testing mod unresolved as a sanity check and its worth the ROI.
   # The resolved would the Fetcher. We have unit tests to cover those other changes.

data/terraspace.gemspec

@@ -9,7 +9,7 @@ Gem::Specification.new do |spec|
   spec.authors       = ["Tung Nguyen"]
   spec.email         = ["tung@boltops.com"]
   spec.summary       = "Terraspace: The Terraspace Framework"
-  spec.homepage      = "https://github.com/boltops-tools/terraspace"
+  spec.homepage      = "https://terraspace.cloud"
   spec.license       = "Apache-2.0"
 
   spec.files         = File.directory?('.git') ? `git ls-files`.split($/) : Dir.glob("**/*")