terraspace 1.1.6 → 2.0.1

data/lib/terraspace/cloud/folder/package.rb

@@ -0,0 +1,33 @@
+require 'zip_folder'
+
+class Terraspace::Cloud::Folder
+  class Package < Base
+    def build
+      copy
+      tidy
+      zip # returns zip path
+    end
+
+    def copy
+      FileUtils.rm_rf(artifacts_path)
+      FileUtils.mkdir_p(File.dirname(artifacts_path))
+
+      expr = "#{@mod.cache_dir}/.terraspace-cache/_cache2/#{@type}/*"
+      Dir.glob(expr).each do |src|
+        dest = "#{artifacts_path}/#{File.basename(src)}"
+        FileUtils.mkdir_p(File.dirname(dest))
+        FileUtils.cp(src, dest)
+      end
+    end
+
+    def tidy
+      Tidy.new(@options).cleanup
+    end
+
+    def zip
+      FileUtils.rm_f(zip_path)
+      ZipFolder.zip(artifacts_path, zip_path)
+      zip_path
+    end
+  end
+end

data/lib/terraspace/cloud/folder/tidy.rb

@@ -0,0 +1,54 @@
+class Terraspace::Cloud::Folder
+  class Tidy < Base
+    def cleanup
+      removals.each do |removal|
+        removal = removal.sub(%r{^/},'') # remove leading slash
+        path = "#{artifacts_path}/#{removal}"
+        rm_rf(path)
+      end
+    end
+
+    def removals
+      removals = always_removals
+      removals += get_removals("#{artifacts_path}/.gitignore")
+      removals = removals.reject do |p|
+        tskeep.find do |keep|
+          p.include?(keep)
+        end
+      end
+      removals.uniq
+    end
+
+    def get_removals(file)
+      path = file
+      return [] unless File.exist?(path)
+
+      removal = File.read(path).split("\n")
+      removal.map {|i| i.strip}.reject {|i| i =~ /^#/ || i.empty?}
+    end
+
+    # We clean out ignored files pretty aggressively. So provide
+    # a way for users to keep files from being cleaned out.
+    def tskeep
+      always_keep = %w[]
+      path = "#{artifacts_path}/.tskeep"
+      return always_keep unless File.exist?(path)
+
+      keep = IO.readlines(path)
+      keep = keep.map {|i| i.strip}.reject { |i| i =~ /^#/ || i.empty? }
+      (always_keep + keep).uniq
+    end
+
+    def rm_rf(path)
+      exists = File.exist?("#{path}/.gitkeep") || File.exist?("#{path}/.keep")
+      return if exists
+
+      FileUtils.rm_rf(path)
+    end
+
+    # These directories will be removed regardless of dir level
+    def always_removals
+      %w[.git spec tmp]
+    end
+  end
+end

data/lib/terraspace/cloud/folder/uploader.rb

@@ -0,0 +1,37 @@
+class Terraspace::Cloud::Folder
+  class Uploader < Base
+    attr_reader :record
+    def upload
+      @record = create_record # set @record for start_plan(uploader.record)
+      upload_project(@record['url'], zip_path)
+    end
+
+    def create_record
+      result = api.create_upload
+      if errors?(result)
+        error_message(result)
+        exit 1 # Consider: raise exception can rescue higher up
+      else
+        load_record(result)
+      end
+    end
+
+    def upload_project(url, path)
+      uri = URI.parse(url)
+      object_content = IO.read(path)
+      resp = Net::HTTP.start(uri.host) do |http|
+        http.send_request(
+          'PUT',
+          uri.request_uri,
+          object_content,
+          'content-type' => ''
+        )
+      end
+      unless resp.code =~ /^20/
+        puts "ERROR: Uploading code"
+        puts "resp.body #{resp.body}"
+        exit 1 # TODO: consider raising error
+      end
+    end
+  end
+end

data/lib/terraspace/cloud/folder.rb

@@ -0,0 +1,11 @@
+module Terraspace::Cloud
+  class Folder < Base
+    def upload_data
+      Package.new(@options).build
+      uploader = Uploader.new(@options)
+      uploader.upload
+      @upload = uploader.record
+    end
+    attr_reader :upload # upload record
+  end
+end

data/lib/terraspace/cloud/plan.rb

@@ -0,0 +1,47 @@
+module Terraspace::Cloud
+  class Plan < Base
+    include Terraspace::CLI::Concerns::PlanPath
+
+    def run
+      return unless record?
+
+      build
+      folder = Folder.new(@options.merge(type: "plan"))
+      upload = folder.upload_data # returns upload record
+      result = api.create_plan(
+        upload_id: upload['id'],
+        stack_uid: upload['stack_id'], # use stack_uid since stack_id is friendly url name
+        plan: stage_attrs,
+      )
+      url = terraspace_cloud_info(result)
+      pr_comment(url)
+    rescue Terraspace::NetworkError => e
+      logger.warn "WARN: #{e.class} #{e.message}"
+      logger.warn "WARN: Unable to save data to Terraspace cloud"
+    end
+
+    def cani?
+      api.create_plan(cani: 1)
+    end
+
+    def build
+      clean_cache2_stage
+      # .terraspace-cache/dev/stacks/demo
+      Dir.chdir(@mod.cache_dir) do
+        plan_dir = File.dirname(plan_path)
+
+        IO.write("#{plan_dir}/plan.log", Terraspace::Logger.logs)
+
+        return unless @success
+        return if File.empty?(plan_path)
+
+        out_option_root_path = "#{Terraspace.root}/#{plan_path}"
+        return unless File.exist?(out_option_root_path)
+        FileUtils.cp(out_option_root_path, plan_path)
+
+        json = plan_path.sub('.binary','.json')
+        sh "terraform show -json #{plan_path} > #{json}"
+      end
+    end
+  end
+end

data/lib/terraspace/cloud/update.rb

@@ -0,0 +1,37 @@
+module Terraspace::Cloud
+  class Update < Base
+    def run
+      return unless record?
+
+      build
+      folder = Folder.new(@options.merge(type: @kind))
+      upload = folder.upload_data # returns upload record
+      result = api.create_update(
+        upload_id: upload['id'],
+        stack_uid: upload['stack_id'], # use stack_uid since stack_id is friendly url name
+        update: stage_attrs,
+      )
+      url = terraspace_cloud_info(result)
+      pr_comment(url)
+    end
+
+    def cani?
+      api.create_update(cani: 1)
+    end
+
+    def build
+      clean_cache2_stage
+      # .terraspace-cache/dev/stacks/demo
+      Dir.chdir(@mod.cache_dir) do
+        cache2_path = ".terraspace-cache/_cache2/#{@kind}"
+        FileUtils.mkdir_p(cache2_path)
+
+        IO.write("#{cache2_path}/#{@kind}.log", Terraspace::Logger.logs)
+        return unless @success
+
+        sh "terraform state pull > #{cache2_path}/state.json"
+        sh "terraform output -json > #{cache2_path}/output.json"
+      end
+    end
+  end
+end

data/lib/terraspace/command.rb

@@ -29,10 +29,25 @@ module Terraspace
     class << self
       include Terraspace::Util::Logging
 
+      @@initial_dispatch_command = nil
+      def initial_dispatch_command
+        @@initial_dispatch_command
+      end
+
+      # use by test framework
+      def reset_dispatch_command
+        @@initial_dispatch_command = nil
+      end
+
       def dispatch(m, args, options, config)
         # Terraspace.argv provides consistency when terraspace is being called by rspec-terraspace test harness
         Terraspace.argv = args.clone # important to clone since Thor removes the first argv
 
+        unless @@initial_dispatch_command
+          @@initial_dispatch_command = "$ terraspace #{args.join(' ')}\n"
+          Terraspace::Logger.buffer << @@initial_dispatch_command
+        end
+
         check_standalone_install!
         check_project!(args.first)
 
@@ -105,7 +120,7 @@ module Terraspace
         return if %w[-h -v --version check_setup completion completion_script help new setup test version].include?(command_name)
         return if File.exist?("#{Terraspace.root}/config/app.rb")
         return unless Terraspace.check_project
-        logger.error "ERROR: It doesnt look like this is a terraspace project. Are you sure you are in a terraspace project?".color(:red)
+        logger.error "ERROR: It doesn't look like this is a terraspace project. Are you sure you are in a terraspace project?".color(:red)
         ENV['TS_TEST'] ? raise : exit(1)
       end
 

data/lib/terraspace/compiler/dsl/syntax/mod.rb

@@ -1,8 +1,8 @@
 module Terraspace::Compiler::Dsl::Syntax
   module Mod
     include Terraspace::Util::Logging
-    include_dir("mod")
-    include_dir("helpers")
+    include_modules("mod")
+    include_modules("helpers")
     include_plugin_helpers
     include_project_level_helpers
   end

data/lib/terraspace/compiler/dsl/syntax/tfvar.rb

@@ -1,6 +1,6 @@
 module Terraspace::Compiler::Dsl::Syntax
   module Tfvar
-    include_dir("tfvar")
+    include_modules("tfvar")
     include Terraspace::Compiler::Dependencies::Helpers
   end
 end

data/lib/terraspace/compiler/expander/backend.rb

@@ -24,7 +24,7 @@ class Terraspace::Compiler::Expander
 
   private
     # Use original unrendered source as wont know the
-    # @mod.cache_dir = ":CACHE_ROOT/:REGION/:ENV/:BUILD_DIR"
+    # @mod.cache_dir = ":REGION/:ENV/:BUILD_DIR"
     # Until the concrete Terraspace Plugin Expander has been autodetected.
     # Follow same precedence rules as rest of Terraspace.
     def src_path

data/lib/terraspace/compiler/expander.rb

@@ -44,7 +44,7 @@ module Terraspace::Compiler
 
     # autodetect by looking up loaded plugins
     def plugin_backend
-      plugin = Terraspace::Autodetect.new.plugin # IE: aws, azurerm, google
+      plugin = Terraspace::Plugin.autodetect
       if plugin
         data = Terraspace::Plugin.meta[plugin]
         data[:backend] # IE: s3, azurerm, gcs

data/lib/terraspace/compiler/strategy/tfvar/layer.rb

@@ -50,21 +50,35 @@ class Terraspace::Compiler::Strategy::Tfvar
     memoize :paths
 
     def full_paths(tfvars_dir)
-      layer_paths = full_layering.map do |layer|
-        [
-          "#{tfvars_dir}/#{layer}.tfvars",
-          "#{tfvars_dir}/#{layer}.rb",
-        ]
-      end.flatten
+      layers = full_layering(tfvars_dir, remove_first: true)
+      if Terraspace.role
+        layers += full_layering("#{tfvars_dir}/#{Terraspace.role}")
+      end
+      if Terraspace.app
+        layers += full_layering("#{tfvars_dir}/#{Terraspace.app}")
+      end
+      if Terraspace.app && Terraspace.role
+        layers += full_layering("#{tfvars_dir}/#{Terraspace.app}/#{Terraspace.role}")
+      end
+      layers
     end
 
-    def full_layering
+    def full_layering(tfvars_dir, remove_first: false)
       # layers defined in Terraspace::Layering module
       all = layers.map { |layer| layer.sub(/\/$/,'') } # strip trailing slash
-      all.inject([]) do |sum, layer|
+      all = all.inject([]) do |sum, layer|
         sum += layer_levels(layer) unless layer.nil?
         sum
       end
+      all.map! do |layer|
+        layer = layer.blank? ? layer : "/#{layer}"
+        [
+          "#{tfvars_dir}#{layer}.tfvars",
+          "#{tfvars_dir}#{layer}.rb",
+        ]
+      end.flatten!
+      all.shift if remove_first # IE: app/stacks/demo/tfvars.tfvars
+      all
     end
 
     # adds prefix and to each layer pair that has base and Terraspace.env. IE:
@@ -73,14 +87,17 @@ class Terraspace::Compiler::Strategy::Tfvar
     #    "#{prefix}/#{Terraspace.env}"
     #
     def layer_levels(prefix=nil)
-      levels = ["base", Terraspace.env, @mod.instance].reject(&:blank?) # layer levels. @mod.instance can be nil
-      env_levels = levels.map { |l| "#{Terraspace.env}/#{l}" } # env folder also
-      levels = levels + env_levels
+      if @mod.instance
+        logger.info "WARN: The instance option is deprecated. Instead use TS_EXTRA"
+        logger.info "See: http://terraspace.test/docs/layering/instance-option/"
+      end
+      extra = Terraspace.extra || @mod.instance
+      levels = ["base", Terraspace.env, extra].reject(&:blank?) # layer levels. @mod.instance can be nil
       levels.map! do |i|
         # base layer has prefix of '', reject with blank so it doesnt produce '//'
         [prefix, i].reject(&:blank?).join('/')
       end
-      levels.unshift(prefix) unless prefix.blank? # IE: tfvars/us-west-2.tfvars
+      levels.unshift(prefix) if !prefix.nil?
       levels
     end
 
@@ -93,18 +110,22 @@ class Terraspace::Compiler::Strategy::Tfvar
         layers << layer.region
 
         namespace = friendly_name(layer.namespace)
-
-        # namespace is a simple way keep different tfvars between different engineers on different accounts
-        layers << namespace
-        layers << "#{namespace}/#{layer.region}"
-
-        # in case using multiple providers and one region
-        layers << layer.provider
-        layers << "#{layer.provider}/#{layer.region}" # also in case another provider has colliding regions
-
-        # Most general layering
-        layers << "#{layer.provider}/#{namespace}"
-        layers << "#{layer.provider}/#{namespace}/#{layer.region}"
+        mode = Terraspace.config.layering.mode # simple, namespace, provider
+        if mode == "namespace" || mode == "provider"
+          # namespace is a simple way keep different tfvars between different engineers on different accounts
+          layers << namespace
+          layers << "#{namespace}/#{layer.region}"
+        end
+
+        if mode == "provider"
+          # in case using multiple providers and one region
+          layers << layer.provider
+          layers << "#{layer.provider}/#{layer.region}" # also in case another provider has colliding regions
+
+          # Most general layering
+          layers << "#{layer.provider}/#{namespace}"
+          layers << "#{layer.provider}/#{namespace}/#{layer.region}"
+        end
       end
       layers
     end
@@ -125,7 +146,7 @@ class Terraspace::Compiler::Strategy::Tfvar
     # should go.
     #
     def stack_tfvars_dir
-      seed_dir = "#{Terraspace.root}/seed/tfvars/#{@mod.build_dir(disable_instance: true)}"
+      seed_dir = "#{Terraspace.root}/seed/tfvars/#{@mod.build_dir(disable_extra: true)}"
       mod_dir = "#{@mod.root}/tfvars"
 
       empty = Dir.glob("#{seed_dir}/*").empty?
@@ -137,14 +158,20 @@ class Terraspace::Compiler::Strategy::Tfvar
       return unless @mod.resolved
       return if @@shown_layers[@mod.name]
       logger.debug "Layers for #{@mod.name}:"
+      show = Terraspace.config.layering.show || ENV['TS_SHOW_ALL_LAYERS']
       paths.each do |path|
+        next if ARGV[0] == "all" # dont show layers with all command since fork happens after build
         next unless path.include?('.tfvars')
-        show = File.exist?(path) || ENV['TS_SHOW_ALL_LAYERS']
-        logger.debug "    #{pretty_path(path)}" if show
+        if ENV['TS_SHOW_ALL_LAYERS']
+          message = "    #{pretty_path(path)}"
+          message = "#{message} (found)".color(:yellow) if File.exist?(path)
+          logger.info message
+        elsif show
+          logger.info "    #{pretty_path(path)}" if File.exist?(path)
+        end
       end
-      logger.debug ""
+      # do not logger.info "" it creates a newline with all
       @@shown_layers[@mod.name] = true
     end
-
   end
 end

data/lib/terraspace/core.rb

@@ -2,10 +2,21 @@ module Terraspace
   module Core
     extend Memoist
 
+    def app
+      ENV['TS_APP'] unless ENV['TS_APP'].blank?
+    end
+
+    def role
+      ENV['TS_ROLE'] unless ENV['TS_ROLE'].blank?
+    end
+
     def env
-      ENV['TS_ENV'] || "dev"
+      ENV['TS_ENV'].blank? ? "dev" : ENV['TS_ENV']
+    end
+
+    def extra
+      ENV['TS_EXTRA'] unless ENV['TS_EXTRA'].blank?
     end
-    memoize :env
 
     @@root = nil
     def root
@@ -15,7 +26,7 @@ module Terraspace
     cattr_writer :root
 
     def cache_root
-      ENV['TS_CACHE_ROOT'] || config.build.cache_root || "#{root}/.terraspace-cache"
+      ENV['TS_CACHE_ROOT'] || "#{root}/.terraspace-cache"
     end
     memoize :cache_root
 
@@ -60,5 +71,27 @@ module Terraspace
     # So use Terraspace.argv instead of ARGV constant
     cattr_accessor :argv
     cattr_accessor :check_project, default: true
+
+    @@cloud_warning_shown = false
+    def cloud?
+      enabled = !!Terraspace.config.cloud.org
+      if ENV['TS_TOKEN'] && !enabled && !@@cloud_warning_shown
+        logger.warn <<~EOL.color(:yellow)
+          WARN: TS_TOKEN is set but config.cloud.org is not set.
+          See: http://terraspace.cloud/docs/cloud/setup/
+        EOL
+        @@cloud_warning_shown = true
+      end
+      enabled
+    end
+
+    @@buffer = []
+    def buffer
+      @@buffer
+    end
+
+    def command?(name)
+      ARGV[0] == name || ARGV[1] == name
+    end
   end
 end

data/lib/terraspace/ext/core/module.rb

@@ -10,17 +10,22 @@ class Module
   #   windows: "C:/Ruby31-x64/lib/ruby/gems/3.1.0/gems/terraspace-1.1.1/lib/terraspace/builder.rb:34:in `build'"
   #   linux: "/home/ec2-user/.rvm/gems/ruby-3.0.3/gems/terraspace-1.1.1/lib/terraspace/compiler/dsl/syntax/mod.rb:4:in `<module:Mod>'"
   #
-  def include_dir(dir)
+  def include_modules(dir)
     caller_line = caller[0]
     parts = caller_line.split(':')
     calling_file = caller_line.match(/^[a-zA-Z]:/) ? parts[1] : parts[0]
     parent_dir = File.dirname(calling_file)
 
     full_dir = "#{parent_dir}/#{dir}"
-    Dir.glob("#{full_dir}/**/*").each do |path|
+    paths = Dir.glob("#{full_dir}/**/*.rb")
+    if paths.empty?
+      raise "Empty include_modules full_dir: #{full_dir}"
+    end
+    paths.each do |path|
       regexp = Regexp.new(".*/lib/")
-      klass = path.sub(regexp, '').sub('.rb','').camelize
-      include klass.constantize
+      mod = path.sub(regexp, '').sub('.rb','').camelize
+      c = mod.constantize
+      include c if c.class == Module
     end
   end
 

data/lib/terraspace/hooks/builder.rb

@@ -40,7 +40,7 @@ module Terraspace::Hooks
       command = File.basename(@file).sub('.rb','') # IE: terraform or terraspace
       id = "#{command} #{type} #{@name}"
       label = " label: #{hook["label"]}" if hook["label"]
-      logger.info  "Hook: Running #{id} hook.#{label}".color(:cyan)
+      logger.info  "Hook: Running #{id} hook.#{label}".color(:cyan) if Terraspace.config.hooks.show
       Runner.new(@mod, hook).run
     end
 

data/lib/terraspace/logger.rb

@@ -14,17 +14,44 @@ module Terraspace
       else
         super # use the configured formatter
       end
-      line =~ /\n$/ ? line : "#{line}\n"
+      out = line =~ /\n$/ ? line : "#{line}\n"
+      @@buffer << out
+      out
     end
 
     # Used to allow terraform output to always go to stdout
     # Terraspace output goes to stderr by default
     # See: terraspace/shell.rb
     def stdout(msg, newline: true)
-      if newline
-        puts msg
-      else
-        print msg
+      out = newline ? "#{msg}\n" : msg
+      @@buffer << out
+      print out
+    end
+
+    def stdin_capture(text)
+      @@buffer << "#{text}\n"
+      @@stdin_capture = text
+    end
+
+    class << self
+      @@stdin_capture = ''
+      def stdin_capture
+        @@stdin_capture
+      end
+
+      @@buffer = []
+      def buffer
+        @@buffer
+      end
+
+      def logs
+        @@buffer.join('')
+      end
+
+      # for test framework
+      def clear
+        Terraspace::Command.reset_dispatch_command
+        @@buffer = []
       end
     end
   end

data/lib/terraspace/mod.rb

@@ -14,7 +14,7 @@ module Terraspace
     def initialize(name, options={})
       @name, @options = placeholder(name), options
       @consider_stacks = options[:consider_stacks].nil? ? true : options[:consider_stacks]
-      @instance = options[:instance]
+      @extra = Terraspace.extra || options[:instance]
       @resolved = true # more common case
     end
 
@@ -72,7 +72,7 @@ module Terraspace
     # If the app/stacks/NAME has been removed in source code but stack still exist in the cloud.
     # allow user to delete by materializing an empty stack with the backend.tf
     # Note this does not seem to work for Terraform Cloud as terraform init doesnt seem to download the plugins
-    # required. SIt only works for s3, azurerm, and gcs backends. On TFC, you can delete the stack via the GUI though.
+    # required. It only works for s3, azurerm, and gcs backends. On TFC, you can delete the stack via the GUI though.
     #
     #   down - so user can delete stacks w/o needing to create an empty app/stacks/demo folder
     #   null - for the terraspace summary command when there are zero stacks.
@@ -99,11 +99,11 @@ module Terraspace
     #
     #     modules/vpc
     #
-    def build_dir(disable_instance: false)
-      if !@instance.nil? && type_dir == "stacks" && !disable_instance
+    def build_dir(disable_extra: false)
+      if !@extra.nil? && type_dir == "stacks" && !disable_extra
         # add _ in front so instance doesnt collide with other default stacks
         # never add for app/modules sources
-        instance_name = [name, @instance].compact.join('.')
+        instance_name = [name, @extra].compact.join('.')
       else
         instance_name = name
       end
@@ -111,9 +111,9 @@ module Terraspace
     end
 
     # Full path with build_dir
-    def cache_dir
+    def cache_dir(options={})
       # config.build.cache_dir is a String or object that respond_to call. IE:
-      #   :CACHE_ROOT/:REGION/:ENV/:BUILD_DIR
+      #   :REGION/:ENV/:BUILD_DIR
       #   CustomBuildDir.call
       # The call method should return a String pattern used for substitutions
       object = Terraspace.config.build.cache_dir
@@ -128,11 +128,23 @@ module Terraspace
           raise "ERROR: config.build.cache_dir is not a String or responds to the .call method."
         end
 
-      expander = Terraspace::Compiler::Expander.autodetect(self)
-      expander.expansion(pattern) # pattern is a String that contains placeholders for substitutions
+      path = expansion(pattern)
+      path = "#{Terraspace.cache_root}/#{path}"
+      path.gsub!(%r{/+},'/') # remove double slashes are more. IE: // -> / Useful since region is '' in generic expander
+      path
     end
     memoize :cache_dir
 
+    def expansion(pattern)
+      @expander ||= Terraspace::Compiler::Expander.autodetect(self)
+      @expander.expansion(pattern) # pattern is a String that contains placeholders for substitutions
+    end
+
+    def out_option
+      expand = Terraspace::Terraform::Args::Expand.new(self, @options)
+      expand.out
+    end
+
     def type
       root.include?("/stacks/") ? "stack" : "module"
     end