terraspace 1.0.6 → 1.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 57c0c4f1c1fce5c1dd8fcd7447f5bc5a82e17d66e5353f4d092d492c137021e5
-  data.tar.gz: 81a4ba08410e898a1dd057625ca556392e746c4b48a3453874472c60424c3949
+  metadata.gz: '0898edc00c5c61938fca0f22e97cd6cc1786e94ddc31f089ce095a75b3543240'
+  data.tar.gz: 9ecef3964e431d2665bf98adacea090cf09a4db0aa91a70d8955a8301444857a
 SHA512:
-  metadata.gz: e629795fd75ad23d343ec28390e76756650f11740c6b380ad3df87cb216f7a998acfea282ebb1a1399343c3db2b5abe452705bb82fd61e76a8cd1a2ca258300b
-  data.tar.gz: 7104217cb56ee3ba2317d4d2abe0d4f760fe35de359903349a97bef4c2de3c92aaff3e0605789fd9e970f8a841a2203dedc9d740d9bddf127d2555a3f3818f30
+  metadata.gz: ef0e86ce8739f13f2d5dfe1aa9caa7cae4a31603e6f365c36ba3d3edab25e98b824babb2077f15a983bc8f9f65146c91cc4c89850717fefb88e9bd258e32aecb
+  data.tar.gz: 71313596f783a4a635bec2e3536e38bdc22d0d4983ca862e1c0f45eeeca898eaa1474da0d25db76b7b8af4c557deb85ee293fb29883f1a376343c16a081244ab

data/CHANGELOG.md

@@ -3,6 +3,25 @@
 All notable changes to this project will be documented in this file.
 This project *loosely tries* to adhere to [Semantic Versioning](http://semver.org/), even before v1.0.
 
+## [1.1.3] - 2022-02-17
+- [#207](https://github.com/boltops-tools/terraspace/pull/207) dont fork when all.concurrency = 1
+
+## [1.1.2] - 2022-02-17
+- [#200](https://github.com/boltops-tools/terraspace/pull/200) fix terraspace typos
+- [#202](https://github.com/boltops-tools/terraspace/pull/202) Windows support: fix include_dir for windows
+- [#203](https://github.com/boltops-tools/terraspace/pull/203) Fix ERB for windows
+- [#204](https://github.com/boltops-tools/terraspace/pull/204) improve file check
+- setup check terraform_bin windows support
+- slight layering improvement strip trailing slash, helps with custom layering
+
+## [1.1.1] - 2022-02-02
+- [#199](https://github.com/boltops-tools/terraspace/pull/199) build required dependent stacks as part of terraspace up
+
+## [1.1.0] - 2022-01-30
+- [#196](https://github.com/boltops-tools/terraspace/pull/196) terraspace all: build modules in batches and only each specific stack
+- [#197](https://github.com/boltops-tools/terraspace/pull/197) all plan --output and all up --plan options
+- simplify starter config/app.rb
+
 ## [1.0.6] - 2022-01-24
 - [#195](https://github.com/boltops-tools/terraspace/pull/195) improve autodetection for plugin expander for backend like remote
 

data/lib/templates/base/project/.gitignore

@@ -5,6 +5,7 @@ terraform.tfvars
 *.tfstate*
 
 .terraspace-cache
+*.plan
 
 # OS X files
 .history

data/lib/templates/base/project/config/app.rb

@@ -1,4 +1,4 @@
+# Docs: https://terraspace.cloud/docs/config/reference/
 Terraspace.configure do |config|
   config.logger.level = :info
-  config.test_framework = "rspec"
 end

data/lib/terraspace/all/grapher.rb

@@ -3,14 +3,13 @@ require "tty-tree"
 
 module Terraspace::All
   class Grapher < Base
+    include Terraspace::Compiler::DirsConcern
     include Terraspace::Util::Logging
 
     def run
       check_graphviz!
       logger.info "Building graph..."
-      builder = Terraspace::Builder.new(@options.merge(mod: "placeholder", quiet: true, draw_full_graph: draw_full_graph))
-      builder.run
-      graph = builder.graph
+      graph = build_graph
       if @options[:format] == "text"
         text(graph.top_nodes)
       else
@@ -18,6 +17,15 @@ module Terraspace::All
       end
     end
 
+    def build_graph
+      resolver = Terraspace::Dependency::Resolver.new(@options.merge(quiet: true, draw_full_graph: draw_full_graph))
+      resolver.resolve
+      dependencies = Terraspace::Dependency::Registry.data # populated after build_unresolved
+      graph = Terraspace::Dependency::Graph.new(stack_names, dependencies, @options)
+      graph.build
+      graph
+    end
+
     def text(nodes)
       Rainbow.enabled = false unless @options[:full]
       data = build_tree_data(nodes)

data/lib/terraspace/all/runner.rb

@@ -1,6 +1,7 @@
 module Terraspace::All
   class Runner < Base
     include Terraspace::Util
+    extend Memoist
 
     def initialize(command, options={})
       @command, @options = command, options
@@ -21,42 +22,56 @@ module Terraspace::All
     end
 
     def build_batches
-      @batches = run_builder(quiet: false)
+      @batches = Terraspace::Dependency::Resolver.new(@options).resolve
       @batches.reverse! if @command == "down"
       @batches
     end
 
     def deploy_batches
       truncate_logs if ENV['TS_TRUNCATE_LOGS']
+      build_modules
       @batches.each_with_index do |batch,i|
         logger.info "Batch Run #{i+1}:"
-        run_builder unless i == 0 # already handled by build_batches the first time
         deploy_batch(batch)
       end
     end
 
-    # Should run after each batch run. run_builder also calls replace_outputs.
-    # Important: rebuild from source so placeholders are in place.
-    def run_builder(quiet: true)
-      Terraspace::Builder.new(@options.merge(mod: "placeholder", quiet: quiet)).run
-    end
-
     def deploy_batch(batch)
       @pids = {} # stores child processes pids. map of pid to mod_name, reset this list on each batch run
       concurrency = Terraspace.config.all.concurrency
       batch.sort_by(&:name).each_slice(concurrency) do |slice|
         slice.each do |node|
-          pid = fork do
-            run_terraspace(node.name)
+          if fork?
+            pid = fork do
+              deploy_stack(node)
+            end
+            @pids[pid] = node.name # store mod_name mapping
+          else
+            deploy_stack(node)
           end
-          @pids[pid] = node.name # store mod_name mapping
         end
       end
+      return unless fork?
       wait_for_child_proccess
       summarize     # also reports lower-level error info
       report_errors # reports finall errors and possibly exit
     end
 
+    def deploy_stack(node)
+      build_stack(node.name)
+      run_terraspace(node.name)
+    end
+
+    def build_modules
+      builder = Terraspace::Builder.new(@options.merge(mod: "placeholder", clean: true, quiet: true, include_stacks: :none))
+      builder.build(modules: true)
+    end
+
+    def build_stack(name)
+      builder = Terraspace::Builder.new(@options.merge(mod: name, clean: false, quiet: true, include_stacks: :root_only))
+      builder.build(modules: false)
+    end
+
     def wait_for_child_proccess
       @errors = [] # stores child processes pids that errored
       @pids.each do |pid, _|
@@ -70,7 +85,7 @@ module Terraspace::All
       @errors.each do |pid|
         mod_name = @pids[pid]
         terraspace_command = terraspace_command(mod_name)
-        logger.error "Error running: #{terraspace_command}. Check logs and fix the error.".color(:red)
+        logger.error "Error running: #{terraspace_command}. Fix the error above or check logs for the error.".color(:red)
       end
       unless @errors.empty?
         exit 2 if exit_on_fail?
@@ -97,12 +112,13 @@ module Terraspace::All
           log_path: log_path(mod_name),
           terraspace_command: terraspace_command(mod_name),
         }
-        Summary.new(data).run
+        # Its possible for log file to not get created if RemoteState::Fetcher#validate! fails
+        Summary.new(data).run if File.exist?(data[:log_path])
       end
     end
 
     def run_terraspace(mod_name)
-      set_log_path!(mod_name)
+      set_log_path!(mod_name) if fork?
       name = command_map(@command)
       o = @options.merge(mod: mod_name, yes: true, build: false, input: false, log_to_stderr: true)
       o.merge!(quiet: false) if @command == "init" # noisy so can filter and summarize output
@@ -116,6 +132,10 @@ module Terraspace::All
       end
     end
 
+    def fork?
+      Terraspace.config.all.concurrency > 1
+    end
+
     def set_log_path!(mod_name)
       command = terraspace_command(mod_name)
       path = log_path(mod_name)

data/lib/terraspace/app.rb

@@ -16,7 +16,8 @@ module Terraspace
       config.all = ActiveSupport::OrderedOptions.new
       config.all.concurrency = 5
       config.all.exit_on_fail = ActiveSupport::OrderedOptions.new
-      config.all.exit_on_fail.down = true
+      config.all.exit_on_fail.down = false
+      config.all.exit_on_fail.plan = true
       config.all.exit_on_fail.up = true
 
       config.allow = ActiveSupport::OrderedOptions.new

data/lib/terraspace/builder/children.rb

@@ -0,0 +1,42 @@
+class Terraspace::Builder
+  class Children
+    include Terraspace::Util::Logging
+
+    def initialize(mod, options={})
+      @mod, @options = mod, options
+      @queue = []
+    end
+
+    def build
+      dependencies = Terraspace::Dependency::Registry.data
+      # Find out if current deploy stack contains dependency
+      found = dependencies.find do |parent_child|
+        parent, _ = parent_child.split(':')
+        parent == @mod.name
+      end
+      return unless found
+
+      # Go down graph children, which are the dependencies to build a queue
+      parent, _ = found.split(':')
+      node = Terraspace::Dependency::Node.find_by(name: parent)
+      build_queue(node)
+
+      logger.debug "Terraspace::Builder::Children @queue #{@queue}"
+
+      # Process queue in reverse order to build leaf nodes first
+      @queue.reverse.each do |node|
+        mod = Terraspace::Mod.new(node.name, @options)
+        Terraspace::Compiler::Perform.new(mod).compile
+      end
+    end
+
+    # Use depth first traversal to build queue
+    def build_queue(node)
+      node.children.each do |child|
+        @queue << child
+        build_queue(child)
+      end
+      @queue
+    end
+  end
+end

data/lib/terraspace/builder.rb

@@ -4,64 +4,66 @@ module Terraspace
     include Compiler::DirsConcern
     include Hooks::Concern
 
-    attr_reader :graph
+    # @include_stacks can be 3 values: root_with_children, none, root_only
+    #
+    #     none: dont build any stacks at all. used by `terraspace all up`
+    #     root_only: only build root stack. used by `terraspace all up`
+    #     root_with_children: build all parent stacks as well as the root stack. normal `terraspace up`
+    #
+    def initialize(options={})
+      super
+      @include_stacks = @options[:include_stacks] || :root_with_children
+    end
 
     def run
       return if @options[:build] == false
       Terraspace::CLI::Setup::Check.check!
+      check_allow!
       @mod.root_module = true
       clean
+      resolve_dependencies if @include_stacks == :root_with_children
+      build
+    end
+
+    def resolve_dependencies
+      resolver = Terraspace::Dependency::Resolver.new(@options.merge(quiet: true))
+      resolver.resolve # returns batches
+    end
+
+    def build(modules: true)
       build_dir = Util.pretty_path(@mod.cache_dir)
       placeholder_stack_message
       logger.info "Building #{build_dir}" unless @options[:quiet] # from terraspace all
-
-      batches = nil
       FileUtils.mkdir_p(@mod.cache_dir) # so terraspace before build hooks work
       run_hooks("terraspace.rb", "build") do
-        check_allow!
-        build_unresolved
-        batches = build_batches
-        build_all
+        build_dir("modules") if modules
+        build_stacks
         logger.info "Built in #{build_dir}" unless @options[:quiet] # from terraspace all
       end
-      batches
     end
 
     def check_allow!
       Allow.new(@mod).check!
     end
 
-    # Builds dependency graph and returns the batches to run
-    def build_batches
-      dependencies = Terraspace::Dependency::Registry.data # populated after build_unresolved
-      @graph = Terraspace::Dependency::Graph.new(stack_names, dependencies, @options)
-      @graph.build
-    end
-
-    def build_all
-      # At this point dependencies have been resolved.
-      Terraspace::Terraform::RemoteState::Fetcher.flush!
-      @resolved = true
-      build_unresolved
-    end
-
-    def build_unresolved
-      build_dir("modules")
-      build_dir("stacks")
-      build_root_module
+    def build_stacks
+      return if @include_stacks == :none
+      build_children_stacks if @include_stacks == :root_with_children
+      Compiler::Perform.new(@mod).compile # @include_stacks :root or :root_with_children
     end
 
-    def build_root_module
-      @mod.resolved = @resolved
-      Compiler::Builder.new(@mod).build
+    # Build stacks that are part of the dependency graph. Because .terraspace-cache folders
+    # need to exist so `terraform state pull` works to get the state info.
+    def build_children_stacks
+      children = Children.new(@mod, @options)
+      children.build
     end
 
     def build_dir(type_dir)
       with_each_mod(type_dir) do |mod|
-        mod.resolved = @resolved
         is_root_module = mod.cache_dir == @mod.cache_dir
-        next if is_root_module # handled by build_root_module
-        Compiler::Builder.new(mod).build
+        next if is_root_module # handled by build_stacks
+        Compiler::Perform.new(mod).compile
       end
     end
 

data/lib/terraspace/cli/all.rb

@@ -38,6 +38,7 @@ class Terraspace::CLI
 
     desc "plan", "Run plan for all or multiple stacks."
     long_desc Help.text("all/plan")
+    option :out, aliases: :o, desc: "Output path. Can be a pattern like :MOD_NAME.plan"
     def plan(*stacks)
       Terraspace::All::Runner.new("plan", @options.merge(stacks: stacks)).run
     end
@@ -56,6 +57,7 @@ class Terraspace::CLI
 
     desc "up", "Deploy all or multiple stacks."
     long_desc Help.text("all/up")
+    option :plan, desc: "Plan path. Can be a pattern like :MOD_NAME.plan"
     def up(*stacks)
       Terraspace::All::Runner.new("up", @options.merge(stacks: stacks)).run
     end

data/lib/terraspace/cli/build/placeholder.rb

@@ -3,6 +3,7 @@
 # It's useful for the summary command.
 module Terraspace::CLI::Build
   class Placeholder
+    include Terraspace::Compiler::DirsConcern
     include Terraspace::Util::Logging
 
     def initialize(options={})
@@ -28,7 +29,8 @@ module Terraspace::CLI::Build
     def find_stack
       stack_paths = Dir.glob("{app,vendor}/stacks/*")
       stack_paths.select! do |path|
-        select = Terraspace::Compiler::Select.new(path)
+        stack_name = extract_stack_name(path)
+        select = Terraspace::Compiler::Select.new(stack_name)
         select.selected?
       end
       mod_path = stack_paths.last

data/lib/terraspace/cli/help/all/plan.md

@@ -22,4 +22,14 @@
     terraspace plan a1:  Plan: 2 to add, 0 to change, 0 to destroy.
     terraspace plan a1:  Changes to Outputs:
     Time took: 11s
-    $
+    $
+
+## Using Plan Outputs
+
+Using plan output path. You can specify an output path for the plan that contains pattern for expansion. Example:
+
+    $ terraspace all plan --out ":MOD_NAME.plan"
+
+You can then use this later in terraspace up:
+
+    $ terraspace all up --plan ":MOD_NAME.plan"

data/lib/terraspace/cli/help/all/up.md

@@ -25,3 +25,13 @@ Once you confirm, Terraspace deploys the batches in parallel. Essentially, Terra
     Time took: 25s
 
 Terraspace provides a reduced-noise summary of the runs. The full logs are also written for further inspection and debugging. The [terraspace log](https://terraspace.cloud/reference/terraspace-log/) command is useful for viewing the logs.
+
+## Using Plans
+
+Using plan output path. You can specify an output path for the plan that contains pattern for expansion. Example:
+
+    $ terraspace all plan --out ":MOD_NAME.plan"
+
+You can then use this later in terraspace up:
+
+    $ terraspace all up --plan ":MOD_NAME.plan"

data/lib/terraspace/cli/help/check_setup.md

@@ -1,9 +1,9 @@
 ## Example
 
     $ terraspace check_setup
-    Detected Terrspace version: 0.3.3
+    Detected Terraspace version: 0.3.3
     Detected Terraform bin: /home/ec2-user/.tfenv/bin/terraform
     Detected Terraform v0.13.2
     Terraspace requires Terraform v0.12.x and above
     You're all set!
-    $
+    $

data/lib/terraspace/cli/help/plan.md

@@ -26,4 +26,12 @@
     can't guarantee that exactly these actions will be performed if
     "terraform apply" is subsequently run.
 
-    $
+    $
+
+Using plan output path. You can specify an output path for the plan. Example:
+
+    $ terraspace plan demo --out "my.plan"
+
+You can then use this later in terraspace up:
+
+    $ terraspace up demo --plan "my.plan"

data/lib/terraspace/cli/help/up.md

@@ -27,4 +27,12 @@
 
     bucket_name = bucket-trusty-marmoset
     Time took: 39s
-    $
+    $
+
+Using plan output path. You can specify an output path for the plan. Example:
+
+    $ terraspace plan demo --out "my.plan"
+
+You can then use this later in terraspace up:
+
+    $ terraspace up demo --plan "my.plan"

data/lib/terraspace/cli/init.rb

@@ -52,7 +52,7 @@ class Terraspace::CLI
       return if meta['Dir'] == '.' # root is already built
 
       remote_mod = Terraspace::Mod::Remote.new(meta, @mod)
-      Terraspace::Compiler::Builder.new(remote_mod).build
+      Terraspace::Compiler::Perform.new(remote_mod).build
     end
 
     def auto?

data/lib/terraspace/cli/new/shim.rb

@@ -50,7 +50,7 @@ class Terraspace::CLI::New
 
   private
     def switch_ruby_version_line
-      rbenv_installed = system("type rbenv 2>&1 > /dev/null")
+      rbenv_installed = system("type rbenv > /dev/null 2>&1")
       if rbenv_installed
         'eval "$(rbenv init -)"'
       end

data/lib/terraspace/cli/setup/check.rb

@@ -11,7 +11,7 @@ class Terraspace::CLI::Setup
 
     # Used for the CLI
     def run
-      puts "Detected Terrspace version: #{Terraspace::VERSION}"
+      puts "Detected Terraspace version: #{Terraspace::VERSION}"
       if terraform_bin
         puts "Detected Terraform bin: #{terraform_bin}"
         puts "Detected #{terraform_version_message}"
@@ -65,8 +65,12 @@ class Terraspace::CLI::Setup
     end
 
     def terraform_bin
-      out = `type terraform 2>&1`.strip
-      return unless $?.success?
+      if Gem.win_platform?
+        out = "is terraform.exe".strip
+      else
+        out = `type terraform 2>&1`.strip
+        return unless $?.success?
+      end
       md = out.match(/is (.*)/)
       md[1] if md
     end

data/lib/terraspace/cli.rb

@@ -9,7 +9,7 @@ module Terraspace
       option :yes, aliases: :y, type: :boolean, desc: "-auto-approve the terraform apply"
     }
     out_option = Proc.new {
-      option :out, aliases: :o, desc: "Write the output to path"
+      option :out, aliases: :o, desc: "Output path. Can be a pattern like :MOD_NAME.plan"
     }
     input_option = Proc.new {
       option :input, type: :boolean, desc: "Ask for input for variables if not directly set."
@@ -55,6 +55,7 @@ module Terraspace
     option :quiet, type: :boolean, desc: "quiet output"
     instance_option.call
     yes_option.call
+    option :clean, type: :boolean, default: nil, desc: "Whether or not clean out .terraspace-cache folder first", hide: true
     def build(mod="placeholder")
       Terraspace::Builder.new(options.merge(mod: mod)).run # building any stack builds them all
     end
@@ -190,7 +191,7 @@ module Terraspace
     desc "show STACK", "Run show."
     long_desc Help.text(:show)
     instance_option.call
-    option :plan, desc: "path to created.plan"
+    option :plan, desc: "Plan path. Can be a pattern like :MOD_NAME.plan"
     def show(mod, *args)
       Commander.new("show", options.merge(mod: mod, args: args)).run
     end

data/lib/terraspace/command.rb

@@ -30,7 +30,7 @@ module Terraspace
       include Terraspace::Util::Logging
 
       def dispatch(m, args, options, config)
-        # Terraspace.argv provides consistency when terraspace is being called by rspec-terrspace test harness
+        # Terraspace.argv provides consistency when terraspace is being called by rspec-terraspace test harness
         Terraspace.argv = args.clone # important to clone since Thor removes the first argv
 
         check_standalone_install!

data/lib/terraspace/compiler/dirs_concern.rb

@@ -37,10 +37,15 @@ module Terraspace::Compiler
     #   path     /home/ec2-user/environment/downloads/infra/app/stacks/demo
     def select_stack?(type_dir, path)
       return true unless type_dir == "stacks"
-      select = Terraspace::Compiler::Select.new(path)
+      stack_name = extract_stack_name(path)
+      select = Terraspace::Compiler::Select.new(stack_name)
       select.selected?
     end
 
+    def extract_stack_name(path)
+      path.sub(%r{.*(app|vendor)/stacks/}, '')
+    end
+
     def local_paths(type_dir)
       dirs("app/#{type_dir}/*") + dirs("vendor/#{type_dir}/*")
     end

data/lib/terraspace/compiler/{builder → perform}/skip.rb

@@ -1,4 +1,4 @@
-class Terraspace::Compiler::Builder
+class Terraspace::Compiler::Perform
   class Skip
     def initialize(mod, src_path)
       @mod, @src_path = mod, src_path

data/lib/terraspace/compiler/{builder.rb → perform.rb}

@@ -1,5 +1,5 @@
 module Terraspace::Compiler
-  class Builder
+  class Perform
     include CommandsConcern
     include Basename
 
@@ -7,44 +7,45 @@ module Terraspace::Compiler
       @mod = mod
     end
 
-    def build
-      build_config
-      build_module if @mod.resolved
-      build_tfvars unless command_is?(:seed) #  avoid dependencies being built and erroring when backend bucket doesnt exist
+    def compile
+      compile_config
+      compile_module if @mod.resolved
+      compile_tfvars
     end
 
-    # build common config files: provider and backend for the root module
-    def build_config
-      return unless build?
-      build_config_terraform
+    # compile common config files: provider and backend for the root module
+    def compile_config
+      return unless compile?
+      compile_config_terraform
     end
 
-    def build_module
+    def compile_module
       with_mod_file do |src_path|
-        build_mod_file(src_path)
+        compile_mod_file(src_path)
       end
     end
 
-    def build_tfvars
-      return unless build?
-      Strategy::Tfvar.new(@mod).run # writer within Strategy to control file ordering
+    def compile_tfvars(write: true)
+      return unless compile?
+      return if command_is?(:seed) # avoid dependencies being built and erroring when backend bucket doesnt exist
+      Strategy::Tfvar.new(@mod).run(write: write) # writer within Strategy to control file ordering
     end
 
   private
-    def build?
+    def compile?
       @mod.type == "stack" || @mod.root_module?
     end
 
-    def build_config_terraform
+    def compile_config_terraform
       expr = "#{Terraspace.root}/config/terraform/**/*"
       search(expr).each do |path|
         next unless File.file?(path)
         next if path.include?('config/terraform/tfvars')
-        build_config_file(basename(path))
+        compile_config_file(basename(path))
       end
     end
 
-    def build_config_file(file)
+    def compile_config_file(file)
       existing = search("#{@mod.root}/#{file}").first
       return if existing && existing.ends_with?(".tf") # do not overwrite existing backend.tf, provider.tf, etc
 
@@ -52,10 +53,10 @@ module Terraspace::Compiler
         src_path = search("#{@mod.root}/#{basename(file)}").first # existing source. IE: backend.rb in module folder
       end
       src_path ||= search("#{Terraspace.root}/config/terraform/#{file}").first
-      build_mod_file(src_path) if src_path
+      compile_mod_file(src_path) if src_path
     end
 
-    def build_mod_file(src_path)
+    def compile_mod_file(src_path)
       content = Strategy::Mod.new(@mod, src_path).run
       Writer.new(@mod, src_path: src_path).write(content)
     end

data/lib/terraspace/compiler/select.rb

@@ -3,9 +3,8 @@ module Terraspace::Compiler
     include Terraspace::App::CallableOption::Concern
     include Terraspace::Util::Logging
 
-    def initialize(path)
-      @path = path
-      @stack_name = extract_stack_name(path)
+    def initialize(stack_name)
+      @stack_name = stack_name
     end
 
     def selected?
@@ -61,10 +60,6 @@ module Terraspace::Compiler
       Terraspace.config
     end
 
-    def extract_stack_name(path)
-      path.sub(%r{.*(app|vendor)/stacks/}, '')
-    end
-
     @@ignore_stacks_deprecation_warning = nil
     def ignore_stacks_deprecation_warning
       return unless config.all.ignore_stacks

data/lib/terraspace/compiler/strategy/mod/text_file.rb

@@ -8,28 +8,29 @@ class Terraspace::Compiler::Strategy::Mod
       @filename = filename
     end
 
-    @@already_reported = false
     def check
-      unless file_installed?
-        return true if @@already_reported
-        logger.warn <<~EOL.color(:yellow)
-          WARN: The command 'file' is not installed.
-          Unable to check if files are text or binary files as a part of the Terraspace compile processing.
-          Assuming all files are not binary file.
+      return true if Gem.win_platform? # assume text file if on windows
+      return true unless file_installed?
 
-          Please install the file command to remove this warning message.
-        EOL
-        @@already_reported = true
-        return true
-      end
       # Thanks: https://stackoverflow.com/questions/2355866/ruby-how-to-determine-if-file-being-read-is-binary-or-text
       file_type, status = Open3.capture2e("file", @filename)
       status.success? && file_type.include?("text")
     end
 
   private
+    @@has_file = nil
     def file_installed?
-      system("type file > /dev/null 2>&1")
+      return @@has_file unless @@has_file.nil?
+      @@has_file = system("type file > /dev/null 2>&1")
+      unless @@has_file
+        logger.warn <<~EOL.color(:yellow)
+          WARN: The command 'file' is not installed.
+          Unable to check if files are text or binary files as a part of the Terraspace compile processing.
+          Assuming all files are not binary file.
+          Please install the file command to remove this warning message.
+        EOL
+      end
+     @@has_file
     end
   end
 end

data/lib/terraspace/compiler/strategy/tfvar/layer.rb

@@ -58,8 +58,9 @@ class Terraspace::Compiler::Strategy::Tfvar
     end
 
     def full_layering
-      # layers is defined in Terraspace::Layering module
-      layers.inject([]) do |sum, layer|
+      # layers defined in Terraspace::Layering module
+      all = layers.map { |layer| layer.sub(/\/$/,'') } # strip trailing slash
+      all.inject([]) do |sum, layer|
         sum += layer_levels(layer) unless layer.nil?
         sum
       end

data/lib/terraspace/compiler/strategy/tfvar.rb

@@ -5,7 +5,7 @@ module Terraspace::Compiler::Strategy
       @order = 0
     end
 
-    def run
+    def run(write: true)
       layer_paths.each do |layer_path|
         ext = File.extname(layer_path).sub('.','')
         klass = strategy_class(ext)
@@ -14,6 +14,7 @@ module Terraspace::Compiler::Strategy
         strategy = klass.new(@mod, layer_path)
         content = strategy.run
 
+        next unless write
         dest_name = ordered_name(layer_path)
         writer = Terraspace::Compiler::Writer.new(@mod, dest_name: dest_name)
         writer.write(content)

data/lib/terraspace/core.rb

@@ -59,7 +59,7 @@ module Terraspace
       end
     end
 
-    # Terraspace.argv provides consistency when terraspace is being called by rspec-terrspace test harness
+    # Terraspace.argv provides consistency when terraspace is being called by rspec-terraspace test harness
     # So use Terraspace.argv instead of ARGV constant
     def argv=(argv)
       @@argv = argv

data/lib/terraspace/dependency/resolver.rb

@@ -0,0 +1,20 @@
+module Terraspace::Dependency
+  class Resolver
+    include Terraspace::Compiler::DirsConcern
+
+    def initialize(options={})
+      @options = options
+    end
+
+    def resolve
+      with_each_mod("stacks") do |mod|
+        mod.resolved = false
+        Terraspace::Compiler::Perform.new(mod).compile_tfvars(write: false)
+      end
+
+      dependencies = Terraspace::Dependency::Registry.data # populated dependencies resolved
+      @graph = Terraspace::Dependency::Graph.new(stack_names, dependencies, @options)
+      @graph.build # Returns batches to run
+    end
+  end
+end

data/lib/terraspace/ext/core/module.rb

@@ -5,8 +5,15 @@ class Module
   #    include Provider
   #    # etc
   #
+  # Caller lines are different for OSes:
+  #
+  #   windows: "C:/Ruby31-x64/lib/ruby/gems/3.1.0/gems/terraspace-1.1.1/lib/terraspace/builder.rb:34:in `build'"
+  #   linux: "/home/ec2-user/.rvm/gems/ruby-3.0.3/gems/terraspace-1.1.1//lib/terraspace/compiler/dsl/syntax/mod.rb:4:in `<module:Mod>'"
+  #
   def include_dir(dir)
-    calling_file = caller[0].split(':').first # IE: /home/ec2-user/environment/terraspace/lib/terraspace/compiler/dsl/syntax/mod.rb
+    caller_line = caller[0]
+    parts = caller_line.split(':')
+    calling_file = caller_line.match(/^[a-zA-Z]:/) ? parts[1] : parts[0]
     parent_dir = File.dirname(calling_file)
 
     full_dir = "#{parent_dir}/#{dir}"

data/lib/terraspace/mod.rb

@@ -15,6 +15,7 @@ module Terraspace
       @name, @options = placeholder(name), options
       @consider_stacks = options[:consider_stacks].nil? ? true : options[:consider_stacks]
       @instance = options[:instance]
+      @resolved = true # more common case
     end
 
     def placeholder(name)

data/lib/terraspace/plugin/summary/interface.rb

@@ -49,7 +49,7 @@ module Terraspace::Plugin::Summary
     def download_statefiles
       return unless download?
       FileUtils.rm_rf(@dest_folder)
-      logger.info("Downloading statefiles to #{@dest_folder}")
+      logger.debug("Downloading statefiles to #{@dest_folder}")
       download # INTERFACE METHOD
     end
 

data/lib/terraspace/terraform/args/expand.rb

@@ -0,0 +1,25 @@
+module Terraspace::Terraform::Args
+  class Expand
+    class << self
+      def option_method(*names)
+        names.compact.each do |name|
+          option_method_each(name)
+        end
+      end
+
+      def option_method_each(name)
+        define_method name do
+          return unless @options[name]
+          expander = Terraspace::Compiler::Expander.autodetect(@mod)
+          expander.expansion(@options[name]) # pattern is a String that contains placeholders for substitutions
+        end
+      end
+    end
+
+    def initialize(mod, options={})
+      @mod, @options = mod, options
+    end
+
+    option_method :plan, :out
+  end
+end

data/lib/terraspace/terraform/args/thor.rb

@@ -2,6 +2,8 @@ require "tempfile"
 
 module Terraspace::Terraform::Args
   class Thor
+    extend Memoist
+
     def initialize(mod, name, options={})
       @mod, @name, @options = mod, name.underscore, options
       @quiet = @options[:quiet].nil? ? true : @options[:quiet]
@@ -48,7 +50,7 @@ module Terraspace::Terraform::Args
       args << input_option
 
       # must be at the end
-      plan = @options[:plan]
+      plan = expand.plan
       if plan
         copy_to_cache(plan)
         args << " #{plan}"
@@ -81,7 +83,7 @@ module Terraspace::Terraform::Args
 
     def output_args
       args = []
-      args << "> #{expanded_out}" if @options[:out]
+      args << "> #{expand.out}" if expand.out
       args
     end
 
@@ -89,26 +91,22 @@ module Terraspace::Terraform::Args
       args = []
       args << input_option
       args << "-destroy" if @options[:destroy]
-      args << "-out #{expanded_out}" if @options[:out]
-      # Note: based on the @options[:out] will run an internal hook to copy plan
+      args << "-out #{expand.out}" if expand.out
+      # Note: based on the expand.out will run an internal hook to copy plan
       # file back up to the root project folder for use. Think this is convenient and expected behavior.
       args
     end
 
     def show_args
       args = []
-      plan = @options[:plan]
+      plan = expand.plan
       if plan
-        copy_to_cache(@options[:plan])
-        args << " #{@options[:plan]}" # terraform show /path/to/plan
+        copy_to_cache(expand.plan)
+        args << " #{expand.plan}" # terraform show /path/to/plan
       end
       args
     end
 
-    def expanded_out
-      @options[:out]
-    end
-
     def destroy_args
       auto_approve_arg
     end
@@ -147,5 +145,10 @@ module Terraspace::Terraform::Args
       FileUtils.mkdir_p(File.dirname(dest))
       FileUtils.cp(src, dest) unless same_file?(src, dest)
     end
+
+    def expand
+      Terraspace::Terraform::Args::Expand.new(@mod, @options)
+    end
+    memoize :expand
   end
 end

data/lib/terraspace/terraform/ihooks/after/plan.rb

@@ -1,8 +1,8 @@
 module Terraspace::Terraform::Ihooks::After
   class Plan < Terraspace::Terraform::Ihooks::Base
     def run
-      return if !@options[:out] || @options[:copy_to_root] == false
-      copy_to_root(@options[:out])
+      return if !out_option || @options[:copy_to_root] == false
+      copy_to_root(out_option)
     end
 
     def copy_to_root(file)

data/lib/terraspace/terraform/ihooks/base.rb

@@ -4,5 +4,10 @@ module Terraspace::Terraform::Ihooks
       @name = name
       super(options)
     end
+
+    def out_option
+      expand = Terraspace::Terraform::Args::Expand.new(@mod, @options)
+      expand.out
+    end
   end
 end

data/lib/terraspace/terraform/ihooks/before/plan.rb

@@ -1,12 +1,10 @@
 module Terraspace::Terraform::Ihooks::Before
   class Plan < Terraspace::Terraform::Ihooks::Base
     def run
-      out = @options[:out]
-      return unless out
-      return if out =~ %r{^/} # not need to create parent dir for copy with absolute path
+      return unless out_option
+      return if out_option =~ %r{^/} # not need to create parent dir for copy with absolute path
 
-      out = @options[:out]
-      name = out.sub("#{Terraspace.root}/",'')
+      name = out_option.sub("#{Terraspace.root}/",'')
       dest = "#{@mod.cache_dir}/#{name}"
       FileUtils.mkdir_p(File.dirname(dest))
     end

data/lib/terraspace/terraform/remote_state/fetcher.rb

@@ -2,6 +2,7 @@ module Terraspace::Terraform::RemoteState
   class Fetcher
     extend Memoist
     include Terraspace::Compiler::CommandsConcern
+    include Terraspace::Compiler::DirsConcern
     include Terraspace::Util::Logging
 
     def initialize(parent, identifier, options={})
@@ -47,7 +48,7 @@ module Terraspace::Terraform::RemoteState
     def output_error(type)
       msg = case type
       when :key_not_found
-        "Output #{@output_key} was not found for the #{@parent.name} tfvars file. Either #{@child.name} stack has not been deployed yet or it does not have this output: #{@output_key}"
+        "Output #{@output_key} was not found for the #{@parent.name} tfvars file. Either #{@child.name} stack has not been deployed yet or it does not have this output: #{@output_key}. Also, if local backend is being used and has been removed/cleaned, then it will also result zero-byte state.json with the 'terraform state pull' used to download the terraform state and output will not be found."
       when :state_not_found
         "Output #{@output_key} could not be looked up for the #{@parent.name} tfvars file. #{@child.name} stack needs to be deployed"
       when :bucket_not_found
@@ -62,7 +63,7 @@ module Terraspace::Terraform::RemoteState
     @@download_shown = false
     def pull
       return if @@pull_successes[cache_key]
-      logger.info "Downloading tfstate files for dependencies defined in tfvars..." unless @@download_shown || @options[:quiet]
+      logger.debug "Downloading tfstate files for dependencies defined in tfvars..." unless @@download_shown || @options[:quiet]
       @@download_shown = true
       logger.debug "Downloading tfstate for stack: #{@child.name}"
 
@@ -130,9 +131,15 @@ module Terraspace::Terraform::RemoteState
 
     # Note we already validate mod exist at the terraform_output helper. This is just in case that logic changes.
     def validate!
-      return if @child.exist?
-      logger.error "ERROR: stack #{@child.name} not found".color(:red)
-      exit 1
+      unless @child.exist?
+        logger.error "ERROR: stack #{@child.name} not found".color(:red)
+        exit 1
+      end
+      select = Terraspace::Compiler::Select.new(@child.name)
+      unless select.selected?
+        logger.error "ERROR: stack #{@child.name} is configured to not be included. IE: config.all.include_stacks or config.all.exclude_stacks".color(:red)
+        exit 1
+      end
     end
 
     # Using debug level because all the tfvar files always get evaluated.

data/lib/terraspace/terraform/remote_state/marker/output.rb

@@ -25,7 +25,7 @@ module Terraspace::Terraform::RemoteState::Marker
     end
 
     def warning
-      logger.warn "WARN: The #{@child_name} stack does not exist".color(:yellow)
+      logger.warn "WARN: The #{@child_name} stack does not exist or is configured to be not included. IE: config.all.include_stacks or config.all.exclude_stacks".color(:yellow)
       caller_line = caller.find { |l| l.include?('.tfvars') }
       return unless caller_line # specs dont have a tfvars file
       source_code = PrettyTracer.new(caller_line).source_code

data/lib/terraspace/version.rb

@@ -1,3 +1,3 @@
 module Terraspace
-  VERSION = "1.0.6"
+  VERSION = "1.1.3"
 end

data/spec/terraspace/all/runner_spec.rb

@@ -1,6 +1,7 @@
 describe Terraspace::All::Runner do
   let(:runner) do
     runner = described_class.new(command: "up", yes: true)
+    allow(runner).to receive(:build_modules)
     allow(runner).to receive(:build_batches).and_return(batches)
     allow(runner).to receive(:preview)
     # Just test to the point of the run_builder and deploy_batch

data/spec/terraspace/compiler/erb/render_spec.rb

@@ -1,6 +1,10 @@
 describe Terraspace::Compiler::Erb::Render do
   let(:render) { described_class.new(mod, src_path) }
-  let(:mod)    { Terraspace::Mod.new("a1") }
+  let(:mod)    do
+    mod = Terraspace::Mod.new("a1")
+    mod.resolved = false
+    mod
+  end
 
   # Only testing mod unresolved as a sanity check and its worth the ROI.
   # The resolved would the Fetcher. We have unit tests to cover those other changes.

data/terraspace.gemspec

@@ -9,9 +9,13 @@ Gem::Specification.new do |spec|
   spec.authors       = ["Tung Nguyen"]
   spec.email         = ["tung@boltops.com"]
   spec.summary       = "Terraspace: The Terraspace Framework"
-  spec.homepage      = "https://github.com/boltops-tools/terraspace"
+  spec.homepage      = "https://terraspace.cloud"
   spec.license       = "Apache-2.0"
 
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://github.com/boltops-tools/terraspace"
+  spec.metadata["changelog_uri"] = "https://github.com/boltops-tools/terraspace/blob/master/CHANGELOG.md"
+
   spec.files         = File.directory?('.git') ? `git ls-files`.split($/) : Dir.glob("**/*")
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: terraspace
 version: !ruby/object:Gem::Version
-  version: 1.0.6
+  version: 1.1.3
 platform: ruby
 authors:
 - Tung Nguyen
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-01-24 00:00:00.000000000 Z
+date: 2022-02-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -490,6 +490,7 @@ files:
 - lib/terraspace/builder/allow/env.rb
 - lib/terraspace/builder/allow/region.rb
 - lib/terraspace/builder/allow/stack.rb
+- lib/terraspace/builder/children.rb
 - lib/terraspace/bundle.rb
 - lib/terraspace/cli.rb
 - lib/terraspace/cli/all.rb
@@ -602,8 +603,6 @@ files:
 - lib/terraspace/cli/up.rb
 - lib/terraspace/command.rb
 - lib/terraspace/compiler/basename.rb
-- lib/terraspace/compiler/builder.rb
-- lib/terraspace/compiler/builder/skip.rb
 - lib/terraspace/compiler/cleaner.rb
 - lib/terraspace/compiler/cleaner/backend_change.rb
 - lib/terraspace/compiler/commands_concern.rb
@@ -633,6 +632,8 @@ files:
 - lib/terraspace/compiler/expander.rb
 - lib/terraspace/compiler/expander/backend.rb
 - lib/terraspace/compiler/helper_extender.rb
+- lib/terraspace/compiler/perform.rb
+- lib/terraspace/compiler/perform/skip.rb
 - lib/terraspace/compiler/select.rb
 - lib/terraspace/compiler/strategy/abstract_base.rb
 - lib/terraspace/compiler/strategy/mod.rb
@@ -655,6 +656,7 @@ files:
 - lib/terraspace/dependency/helper/output.rb
 - lib/terraspace/dependency/node.rb
 - lib/terraspace/dependency/registry.rb
+- lib/terraspace/dependency/resolver.rb
 - lib/terraspace/dotenv.rb
 - lib/terraspace/ext.rb
 - lib/terraspace/ext/bundler.rb
@@ -703,6 +705,7 @@ files:
 - lib/terraspace/terraform/api/workspace.rb
 - lib/terraspace/terraform/args/custom.rb
 - lib/terraspace/terraform/args/dsl.rb
+- lib/terraspace/terraform/args/expand.rb
 - lib/terraspace/terraform/args/pass.rb
 - lib/terraspace/terraform/args/shorthands.rb
 - lib/terraspace/terraform/args/thor.rb
@@ -864,10 +867,13 @@ files:
 - spec/terraspace/terraform/remote_state/marker/output_spec.rb
 - spec/terraspace/terraform/remote_state/output_proxy_spec.rb
 - terraspace.gemspec
-homepage: https://github.com/boltops-tools/terraspace
+homepage: https://terraspace.cloud
 licenses:
 - Apache-2.0
-metadata: {}
+metadata:
+  homepage_uri: https://terraspace.cloud
+  source_code_uri: https://github.com/boltops-tools/terraspace
+  changelog_uri: https://github.com/boltops-tools/terraspace/blob/master/CHANGELOG.md
 post_install_message: 
 rdoc_options: []
 require_paths: