terraspace 1.1.7 → 2.0.0

data/lib/terraspace/cloud/folder.rb

@@ -0,0 +1,11 @@
+module Terraspace::Cloud
+  class Folder < Base
+    def upload_data
+      Package.new(@options).build
+      uploader = Uploader.new(@options)
+      uploader.upload
+      @upload = uploader.record
+    end
+    attr_reader :upload # upload record
+  end
+end

data/lib/terraspace/cloud/plan.rb

@@ -0,0 +1,47 @@
+module Terraspace::Cloud
+  class Plan < Base
+    include Terraspace::CLI::Concerns::PlanPath
+
+    def run
+      return unless record?
+
+      build
+      folder = Folder.new(@options.merge(type: "plan"))
+      upload = folder.upload_data # returns upload record
+      result = api.create_plan(
+        upload_id: upload['id'],
+        stack_uid: upload['stack_id'], # use stack_uid since stack_id is friendly url name
+        plan: stage_attrs,
+      )
+      url = terraspace_cloud_info(result)
+      pr_comment(url)
+    rescue Terraspace::NetworkError => e
+      logger.warn "WARN: #{e.class} #{e.message}"
+      logger.warn "WARN: Unable to save data to Terraspace cloud"
+    end
+
+    def cani?
+      api.create_plan(cani: 1)
+    end
+
+    def build
+      clean_cache2_stage
+      # .terraspace-cache/dev/stacks/demo
+      Dir.chdir(@mod.cache_dir) do
+        plan_dir = File.dirname(plan_path)
+
+        IO.write("#{plan_dir}/plan.log", Terraspace::Logger.logs)
+
+        return unless @success
+        return if File.empty?(plan_path)
+
+        out_option_root_path = "#{Terraspace.root}/#{plan_path}"
+        return unless File.exist?(out_option_root_path)
+        FileUtils.cp(out_option_root_path, plan_path)
+
+        json = plan_path.sub('.binary','.json')
+        sh "terraform show -json #{plan_path} > #{json}"
+      end
+    end
+  end
+end

data/lib/terraspace/cloud/update.rb

@@ -0,0 +1,37 @@
+module Terraspace::Cloud
+  class Update < Base
+    def run
+      return unless record?
+
+      build
+      folder = Folder.new(@options.merge(type: @kind))
+      upload = folder.upload_data # returns upload record
+      result = api.create_update(
+        upload_id: upload['id'],
+        stack_uid: upload['stack_id'], # use stack_uid since stack_id is friendly url name
+        update: stage_attrs,
+      )
+      url = terraspace_cloud_info(result)
+      pr_comment(url)
+    end
+
+    def cani?
+      api.create_update(cani: 1)
+    end
+
+    def build
+      clean_cache2_stage
+      # .terraspace-cache/dev/stacks/demo
+      Dir.chdir(@mod.cache_dir) do
+        cache2_path = ".terraspace-cache/_cache2/#{@kind}"
+        FileUtils.mkdir_p(cache2_path)
+
+        IO.write("#{cache2_path}/#{@kind}.log", Terraspace::Logger.logs)
+        return unless @success
+
+        sh "terraform state pull > #{cache2_path}/state.json"
+        sh "terraform output -json > #{cache2_path}/output.json"
+      end
+    end
+  end
+end

data/lib/terraspace/command.rb

@@ -29,10 +29,25 @@ module Terraspace
     class << self
       include Terraspace::Util::Logging
 
+      @@initial_dispatch_command = nil
+      def initial_dispatch_command
+        @@initial_dispatch_command
+      end
+
+      # use by test framework
+      def reset_dispatch_command
+        @@initial_dispatch_command = nil
+      end
+
       def dispatch(m, args, options, config)
         # Terraspace.argv provides consistency when terraspace is being called by rspec-terraspace test harness
         Terraspace.argv = args.clone # important to clone since Thor removes the first argv
 
+        unless @@initial_dispatch_command
+          @@initial_dispatch_command = "$ terraspace #{args.join(' ')}\n"
+          Terraspace::Logger.buffer << @@initial_dispatch_command
+        end
+
         check_standalone_install!
         check_project!(args.first)
 
@@ -105,7 +120,7 @@ module Terraspace
         return if %w[-h -v --version check_setup completion completion_script help new setup test version].include?(command_name)
         return if File.exist?("#{Terraspace.root}/config/app.rb")
         return unless Terraspace.check_project
-        logger.error "ERROR: It doesnt look like this is a terraspace project. Are you sure you are in a terraspace project?".color(:red)
+        logger.error "ERROR: It doesn't look like this is a terraspace project. Are you sure you are in a terraspace project?".color(:red)
         ENV['TS_TEST'] ? raise : exit(1)
       end
 

data/lib/terraspace/compiler/dsl/syntax/mod.rb

@@ -1,8 +1,8 @@
 module Terraspace::Compiler::Dsl::Syntax
   module Mod
     include Terraspace::Util::Logging
-    include_dir("mod")
-    include_dir("helpers")
+    include_modules("mod")
+    include_modules("helpers")
     include_plugin_helpers
     include_project_level_helpers
   end

data/lib/terraspace/compiler/dsl/syntax/tfvar.rb

@@ -1,6 +1,6 @@
 module Terraspace::Compiler::Dsl::Syntax
   module Tfvar
-    include_dir("tfvar")
+    include_modules("tfvar")
     include Terraspace::Compiler::Dependencies::Helpers
   end
 end

data/lib/terraspace/compiler/expander/backend.rb

@@ -24,7 +24,7 @@ class Terraspace::Compiler::Expander
 
   private
     # Use original unrendered source as wont know the
-    # @mod.cache_dir = ":CACHE_ROOT/:REGION/:ENV/:BUILD_DIR"
+    # @mod.cache_dir = ":REGION/:ENV/:BUILD_DIR"
     # Until the concrete Terraspace Plugin Expander has been autodetected.
     # Follow same precedence rules as rest of Terraspace.
     def src_path

data/lib/terraspace/compiler/expander.rb

@@ -44,7 +44,7 @@ module Terraspace::Compiler
 
     # autodetect by looking up loaded plugins
     def plugin_backend
-      plugin = Terraspace::Autodetect.new.plugin # IE: aws, azurerm, google
+      plugin = Terraspace::Plugin.autodetect
       if plugin
         data = Terraspace::Plugin.meta[plugin]
         data[:backend] # IE: s3, azurerm, gcs

data/lib/terraspace/compiler/strategy/tfvar/layer.rb

@@ -50,21 +50,35 @@ class Terraspace::Compiler::Strategy::Tfvar
     memoize :paths
 
     def full_paths(tfvars_dir)
-      layer_paths = full_layering.map do |layer|
-        [
-          "#{tfvars_dir}/#{layer}.tfvars",
-          "#{tfvars_dir}/#{layer}.rb",
-        ]
-      end.flatten
+      layers = full_layering(tfvars_dir, remove_first: true)
+      if Terraspace.role
+        layers += full_layering("#{tfvars_dir}/#{Terraspace.role}")
+      end
+      if Terraspace.app
+        layers += full_layering("#{tfvars_dir}/#{Terraspace.app}")
+      end
+      if Terraspace.app && Terraspace.role
+        layers += full_layering("#{tfvars_dir}/#{Terraspace.app}/#{Terraspace.role}")
+      end
+      layers
     end
 
-    def full_layering
+    def full_layering(tfvars_dir, remove_first: false)
       # layers defined in Terraspace::Layering module
       all = layers.map { |layer| layer.sub(/\/$/,'') } # strip trailing slash
-      all.inject([]) do |sum, layer|
+      all = all.inject([]) do |sum, layer|
         sum += layer_levels(layer) unless layer.nil?
         sum
       end
+      all.map! do |layer|
+        layer = layer.blank? ? layer : "/#{layer}"
+        [
+          "#{tfvars_dir}#{layer}.tfvars",
+          "#{tfvars_dir}#{layer}.rb",
+        ]
+      end.flatten!
+      all.shift if remove_first # IE: app/stacks/demo/tfvars.tfvars
+      all
     end
 
     # adds prefix and to each layer pair that has base and Terraspace.env. IE:
@@ -73,14 +87,17 @@ class Terraspace::Compiler::Strategy::Tfvar
     #    "#{prefix}/#{Terraspace.env}"
     #
     def layer_levels(prefix=nil)
-      levels = ["base", Terraspace.env, @mod.instance].reject(&:blank?) # layer levels. @mod.instance can be nil
-      env_levels = levels.map { |l| "#{Terraspace.env}/#{l}" } # env folder also
-      levels = levels + env_levels
+      if @mod.instance
+        logger.info "WARN: The instance option is deprecated. Instead use TS_EXTRA"
+        logger.info "See: http://terraspace.test/docs/layering/instance-option/"
+      end
+      extra = Terraspace.extra || @mod.instance
+      levels = ["base", Terraspace.env, extra].reject(&:blank?) # layer levels. @mod.instance can be nil
       levels.map! do |i|
         # base layer has prefix of '', reject with blank so it doesnt produce '//'
         [prefix, i].reject(&:blank?).join('/')
       end
-      levels.unshift(prefix) unless prefix.blank? # IE: tfvars/us-west-2.tfvars
+      levels.unshift(prefix) if !prefix.nil?
       levels
     end
 
@@ -93,18 +110,22 @@ class Terraspace::Compiler::Strategy::Tfvar
         layers << layer.region
 
         namespace = friendly_name(layer.namespace)
-
-        # namespace is a simple way keep different tfvars between different engineers on different accounts
-        layers << namespace
-        layers << "#{namespace}/#{layer.region}"
-
-        # in case using multiple providers and one region
-        layers << layer.provider
-        layers << "#{layer.provider}/#{layer.region}" # also in case another provider has colliding regions
-
-        # Most general layering
-        layers << "#{layer.provider}/#{namespace}"
-        layers << "#{layer.provider}/#{namespace}/#{layer.region}"
+        mode = Terraspace.config.layering.mode # simple, namespace, provider
+        if mode == "namespace" || mode == "provider"
+          # namespace is a simple way keep different tfvars between different engineers on different accounts
+          layers << namespace
+          layers << "#{namespace}/#{layer.region}"
+        end
+
+        if mode == "provider"
+          # in case using multiple providers and one region
+          layers << layer.provider
+          layers << "#{layer.provider}/#{layer.region}" # also in case another provider has colliding regions
+
+          # Most general layering
+          layers << "#{layer.provider}/#{namespace}"
+          layers << "#{layer.provider}/#{namespace}/#{layer.region}"
+        end
       end
       layers
     end
@@ -125,7 +146,7 @@ class Terraspace::Compiler::Strategy::Tfvar
     # should go.
     #
     def stack_tfvars_dir
-      seed_dir = "#{Terraspace.root}/seed/tfvars/#{@mod.build_dir(disable_instance: true)}"
+      seed_dir = "#{Terraspace.root}/seed/tfvars/#{@mod.build_dir(disable_extra: true)}"
       mod_dir = "#{@mod.root}/tfvars"
 
       empty = Dir.glob("#{seed_dir}/*").empty?
@@ -137,14 +158,20 @@ class Terraspace::Compiler::Strategy::Tfvar
       return unless @mod.resolved
       return if @@shown_layers[@mod.name]
       logger.debug "Layers for #{@mod.name}:"
+      show = Terraspace.config.layering.show || ENV['TS_SHOW_ALL_LAYERS']
       paths.each do |path|
+        next if ARGV[0] == "all" # dont show layers with all command since fork happens after build
         next unless path.include?('.tfvars')
-        show = File.exist?(path) || ENV['TS_SHOW_ALL_LAYERS']
-        logger.debug "    #{pretty_path(path)}" if show
+        if ENV['TS_SHOW_ALL_LAYERS']
+          message = "    #{pretty_path(path)}"
+          message = "#{message} (found)".color(:yellow) if File.exist?(path)
+          logger.info message
+        elsif show
+          logger.info "    #{pretty_path(path)}" if File.exist?(path)
+        end
       end
-      logger.debug ""
+      # do not logger.info "" it creates a newline with all
       @@shown_layers[@mod.name] = true
     end
-
   end
 end

data/lib/terraspace/core.rb

@@ -2,10 +2,21 @@ module Terraspace
   module Core
     extend Memoist
 
+    def app
+      ENV['TS_APP'] unless ENV['TS_APP'].blank?
+    end
+
+    def role
+      ENV['TS_ROLE'] unless ENV['TS_ROLE'].blank?
+    end
+
     def env
-      ENV['TS_ENV'] || "dev"
+      ENV['TS_ENV'].blank? ? "dev" : ENV['TS_ENV']
+    end
+
+    def extra
+      ENV['TS_EXTRA'] unless ENV['TS_EXTRA'].blank?
     end
-    memoize :env
 
     @@root = nil
     def root
@@ -15,7 +26,7 @@ module Terraspace
     cattr_writer :root
 
     def cache_root
-      ENV['TS_CACHE_ROOT'] || config.build.cache_root || "#{root}/.terraspace-cache"
+      ENV['TS_CACHE_ROOT'] || "#{root}/.terraspace-cache"
     end
     memoize :cache_root
 
@@ -60,5 +71,27 @@ module Terraspace
     # So use Terraspace.argv instead of ARGV constant
     cattr_accessor :argv
     cattr_accessor :check_project, default: true
+
+    @@cloud_warning_shown = false
+    def cloud?
+      enabled = !!Terraspace.config.cloud.org
+      if ENV['TS_TOKEN'] && !enabled && !@@cloud_warning_shown
+        logger.warn <<~EOL.color(:yellow)
+          WARN: TS_TOKEN is set but config.cloud.org is not set.
+          See: http://terraspace.cloud/docs/cloud/setup/
+        EOL
+        @@cloud_warning_shown = true
+      end
+      enabled
+    end
+
+    @@buffer = []
+    def buffer
+      @@buffer
+    end
+
+    def command?(name)
+      ARGV[0] == name || ARGV[1] == name
+    end
   end
 end

data/lib/terraspace/ext/core/module.rb

@@ -10,17 +10,22 @@ class Module
   #   windows: "C:/Ruby31-x64/lib/ruby/gems/3.1.0/gems/terraspace-1.1.1/lib/terraspace/builder.rb:34:in `build'"
   #   linux: "/home/ec2-user/.rvm/gems/ruby-3.0.3/gems/terraspace-1.1.1/lib/terraspace/compiler/dsl/syntax/mod.rb:4:in `<module:Mod>'"
   #
-  def include_dir(dir)
+  def include_modules(dir)
     caller_line = caller[0]
     parts = caller_line.split(':')
     calling_file = caller_line.match(/^[a-zA-Z]:/) ? parts[1] : parts[0]
     parent_dir = File.dirname(calling_file)
 
     full_dir = "#{parent_dir}/#{dir}"
-    Dir.glob("#{full_dir}/**/*").each do |path|
+    paths = Dir.glob("#{full_dir}/**/*.rb")
+    if paths.empty?
+      raise "Empty include_modules full_dir: #{full_dir}"
+    end
+    paths.each do |path|
       regexp = Regexp.new(".*/lib/")
-      klass = path.sub(regexp, '').sub('.rb','').camelize
-      include klass.constantize
+      mod = path.sub(regexp, '').sub('.rb','').camelize
+      c = mod.constantize
+      include c if c.class == Module
     end
   end
 

data/lib/terraspace/hooks/builder.rb

@@ -40,7 +40,7 @@ module Terraspace::Hooks
       command = File.basename(@file).sub('.rb','') # IE: terraform or terraspace
       id = "#{command} #{type} #{@name}"
       label = " label: #{hook["label"]}" if hook["label"]
-      logger.info  "Hook: Running #{id} hook.#{label}".color(:cyan)
+      logger.info  "Hook: Running #{id} hook.#{label}".color(:cyan) if Terraspace.config.hooks.show
       Runner.new(@mod, hook).run
     end
 

data/lib/terraspace/logger.rb

@@ -14,17 +14,44 @@ module Terraspace
       else
         super # use the configured formatter
       end
-      line =~ /\n$/ ? line : "#{line}\n"
+      out = line =~ /\n$/ ? line : "#{line}\n"
+      @@buffer << out
+      out
     end
 
     # Used to allow terraform output to always go to stdout
     # Terraspace output goes to stderr by default
     # See: terraspace/shell.rb
     def stdout(msg, newline: true)
-      if newline
-        puts msg
-      else
-        print msg
+      out = newline ? "#{msg}\n" : msg
+      @@buffer << out
+      print out
+    end
+
+    def stdin_capture(text)
+      @@buffer << "#{text}\n"
+      @@stdin_capture = text
+    end
+
+    class << self
+      @@stdin_capture = ''
+      def stdin_capture
+        @@stdin_capture
+      end
+
+      @@buffer = []
+      def buffer
+        @@buffer
+      end
+
+      def logs
+        @@buffer.join('')
+      end
+
+      # for test framework
+      def clear
+        Terraspace::Command.reset_dispatch_command
+        @@buffer = []
       end
     end
   end

data/lib/terraspace/mod.rb

@@ -14,7 +14,7 @@ module Terraspace
     def initialize(name, options={})
       @name, @options = placeholder(name), options
       @consider_stacks = options[:consider_stacks].nil? ? true : options[:consider_stacks]
-      @instance = options[:instance]
+      @extra = Terraspace.extra || options[:instance]
       @resolved = true # more common case
     end
 
@@ -72,7 +72,7 @@ module Terraspace
     # If the app/stacks/NAME has been removed in source code but stack still exist in the cloud.
     # allow user to delete by materializing an empty stack with the backend.tf
     # Note this does not seem to work for Terraform Cloud as terraform init doesnt seem to download the plugins
-    # required. SIt only works for s3, azurerm, and gcs backends. On TFC, you can delete the stack via the GUI though.
+    # required. It only works for s3, azurerm, and gcs backends. On TFC, you can delete the stack via the GUI though.
     #
     #   down - so user can delete stacks w/o needing to create an empty app/stacks/demo folder
     #   null - for the terraspace summary command when there are zero stacks.
@@ -99,11 +99,11 @@ module Terraspace
     #
     #     modules/vpc
     #
-    def build_dir(disable_instance: false)
-      if !@instance.nil? && type_dir == "stacks" && !disable_instance
+    def build_dir(disable_extra: false)
+      if !@extra.nil? && type_dir == "stacks" && !disable_extra
         # add _ in front so instance doesnt collide with other default stacks
         # never add for app/modules sources
-        instance_name = [name, @instance].compact.join('.')
+        instance_name = [name, @extra].compact.join('.')
       else
         instance_name = name
       end
@@ -111,9 +111,9 @@ module Terraspace
     end
 
     # Full path with build_dir
-    def cache_dir
+    def cache_dir(options={})
       # config.build.cache_dir is a String or object that respond_to call. IE:
-      #   :CACHE_ROOT/:REGION/:ENV/:BUILD_DIR
+      #   :REGION/:ENV/:BUILD_DIR
       #   CustomBuildDir.call
       # The call method should return a String pattern used for substitutions
       object = Terraspace.config.build.cache_dir
@@ -128,11 +128,23 @@ module Terraspace
           raise "ERROR: config.build.cache_dir is not a String or responds to the .call method."
         end
 
-      expander = Terraspace::Compiler::Expander.autodetect(self)
-      expander.expansion(pattern) # pattern is a String that contains placeholders for substitutions
+      path = expansion(pattern)
+      path = "#{Terraspace.cache_root}/#{path}"
+      path.gsub!(%r{/+},'/') # remove double slashes are more. IE: // -> / Useful since region is '' in generic expander
+      path
     end
     memoize :cache_dir
 
+    def expansion(pattern)
+      @expander ||= Terraspace::Compiler::Expander.autodetect(self)
+      @expander.expansion(pattern) # pattern is a String that contains placeholders for substitutions
+    end
+
+    def out_option
+      expand = Terraspace::Terraform::Args::Expand.new(self, @options)
+      expand.out
+    end
+
     def type
       root.include?("/stacks/") ? "stack" : "module"
     end

data/lib/terraspace/plugin/expander/interface.rb

@@ -58,7 +58,7 @@ module Terraspace::Plugin::Expander
     #
     # cache_dir:
     #
-    #    :CACHE_ROOT/:REGION/:ENV/:BUILD_DIR/
+    #    :REGION/:ENV/:BUILD_DIR/
     #
     # s3 backend key:
     #
@@ -69,21 +69,24 @@ module Terraspace::Plugin::Expander
     #    :MOD_NAME-:ENV-:REGION-:INSTANCE
     #
     def strip(string)
-      string.sub(/^-+/,'').sub(/-+$/,'') # remove leading and trailing -
-            .sub(%r{/+$},'')  # only remove trailing / or else /home/ec2-user => home/ec2-user
+      string.sub(%r{/+$},'')  # only remove trailing / or else /home/ec2-user => home/ec2-user
             .sub(/:\/\//, 'TMP_KEEP_HTTP') # so we can keep ://. IE: https:// or http://
             .gsub(%r{/+},'/') # remove double slashes are more. IE: // -> / Useful of region is '' in generic expander
             .sub('TMP_KEEP_HTTP', '://')   # restore :// IE: https:// or http://
+            .sub(/^-+/,'').sub(/-+$/,'') # remove leading and trailing -
     end
 
     def var_value(unexpanded)
-      name = unexpanded.sub(':','').downcase
-      if respond_to?(name)
-        value = send(name).to_s
+      name = unexpanded.sub(':','')
+      downcase = name.downcase
+      if respond_to?(downcase)
+        value = send(downcase).to_s
+      elsif !ENV[name].blank?
+        return ENV[name]
       else
         return unexpanded
       end
-      if name == "namespace" && Terraspace.config.layering.enable_names.expansion
+      if downcase == "namespace" && Terraspace.config.layering.enable_names.expansion
         value = friendly_name(value)
       end
       value
@@ -93,9 +96,10 @@ module Terraspace::Plugin::Expander
       @mod.name
     end
 
-    def env
-      Terraspace.env
-    end
+    def app;   Terraspace.app   ; end
+    def role;  Terraspace.role  ; end
+    def env;   Terraspace.env   ; end
+    def extra; Terraspace.extra ; end
 
     def type_instance
       [type, instance].reject { |s| s.blank? }.join('-')
@@ -111,7 +115,7 @@ module Terraspace::Plugin::Expander
     end
 
     # So default config works:
-    #    config.cache_dir = ":CACHE_ROOT/:REGION/:ENV/:BUILD_DIR"
+    #    config.cache_dir = ":REGION/:ENV/:BUILD_DIR"
     # For when folks configure it with the http backend for non-cloud providers
     # The double slash // will be replace with a single slash in expander/interface.rb
     def region

data/lib/terraspace/plugin.rb

@@ -7,7 +7,7 @@ module Terraspace
     # Example meta:
     #
     #    {
-    #      "aws  => {root: "/path", backend: "s3"}
+    #      "aws"    => {root: "/path", backend: "s3"}
     #      "google" => {root: "/path", backend: "gcs"},
     #    }
     #
@@ -16,6 +16,10 @@ module Terraspace
       @@meta
     end
 
+    def register(plugin, data)
+      @@meta[plugin] = data
+    end
+
     def config_classes
       @@meta.map { |plugin, data| data[:config_class] }.compact
     end
@@ -50,10 +54,6 @@ module Terraspace
       end
     end
 
-    def register(plugin, data)
-      @@meta[plugin] = data
-    end
-
     # Example return:
     #
     #      TerraspacePluginAws::Interfaces::Backend
@@ -72,6 +72,15 @@ module Terraspace
     end
     memoize :find_with
 
+    # Returns simple String: aws
+    def autodetect
+      plugins = meta.keys
+      precedence = %w[aws azurerm google]
+      precedence.find do |p|
+        plugins.include?(p)
+      end
+    end
+
     extend self
   end
 end

data/lib/terraspace/shell.rb

@@ -34,6 +34,11 @@ module Terraspace
 
     def popen3(env)
       Open3.popen3(env, @command, chdir: @mod.cache_dir) do |stdin, stdout, stderr, wait_thread|
+        # interesting. simply handling the trap and doing nothing works
+        # Think its because ctrl-c is sent to both processes.
+        # 1. we do nothing in here in the parent process
+        # 2. in the child process the ctrl-c gets sent directly to the terraform command
+        Signal.trap("INT") { }
         handle_streams(stdin, stdout, stderr)
         status = wait_thread.value.exitstatus
         exit_status(status)
@@ -112,6 +117,7 @@ module Terraspace
       end
       if matched
         answer = $stdin.gets
+        logger.stdin_capture(answer.strip)
         stdin.write_nonblock(answer)
       end
     end
@@ -139,8 +145,7 @@ module Terraspace
       if @error && @error.known?
         raise @error.instance
       elsif exit_on_fail
-        logger.error "Error running command: #{@command}".color(:red)
-        exit status
+        raise ShellError.new("Error running command: #{@command}")
       end
     end
   end