stack_master 2.7.0 → 2.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f94d2bab69225428425d5e3bc426ca572c781aa5d9e069420c198c86640f84c5
-  data.tar.gz: f4f164e4390d568e9a7511013a33796ad04862d18510ec5683c2024b9952781f
+  metadata.gz: 02eda4742884cf2aa48568ad0e330b375f7548f12f3077be3167bd782a51868d
+  data.tar.gz: 6f3159eb1b681bda06e4c128d4746f05484a75f91c1a2119d9c02523f912b48f
 SHA512:
-  metadata.gz: 81122997265f0aa25d24f54ee3ccbb5a0a3689b362b37445bd0c12435dc0fcbb24e0d8ce277c949c1e3af5c01d3b5ff938227f6b6f051b97cc8da66f400519d6
-  data.tar.gz: b5f8114a90fc7cc402e07e34676b7142b3849c4e8eecdc43cb67651358e5e7e6228ab04b68ef5b769b98ff91e7efd2ec9667a5c197699bccbfb96d49848585ed
+  metadata.gz: 4670acd897af5bd58083f65187c23c7e7add969ca1d906899ce18065bcc24287d78854bd67baa82a1225dab4277b3392fa03cc3a5ffe49c13cffd3665ca2ea99
+  data.tar.gz: 8ea47d21747c13e9685f0f5cacbb12663c28df649823981c5e1a94e982bc7798bd2b93828be921e7e2e4267e3c3aa3d1f246aeb8f99f0eb91d34d2ce24594263

data/README.md

@@ -143,7 +143,8 @@ stacks:
 ## Templates
 
 StackMaster supports CloudFormation templates in plain JSON or YAML. Any `.yml` or `.yaml` file will be processed as
-YAML, while any `.json` file will be processed as JSON.
+YAML, while any `.json` file will be processed as JSON. Additionally, YAML files can be pre-processed using ERB and
+compile-time parameters.
 
 ### Ruby DSLs
 By default, any template ending with `.rb` will be processed as a [SparkleFormation](https://github.com/sparkleformation/sparkle_formation)
@@ -199,12 +200,13 @@ stacks:
 
 ### Compile Time Parameters
 
-Compile time parameters can be used for [SparkleFormation](http://www.sparkleformation.io) templates. It conforms and
-allows you to use the [Compile Time Parameters](http://www.sparkleformation.io/docs/sparkle_formation/compile-time-parameters.html) feature.
+Compile time parameters can be defined in a stack's parameters file, using the key `compile_time_parameters`. Keys in
+parameter files are automatically converted to camel case.
 
-A simple example looks like this
+As an example:
 
 ```yaml
+# parameters/some_stack.yml
 vpc_cidr: 10.0.0.0/16
 compile_time_parameters:
   subnet_cidrs:
@@ -212,7 +214,37 @@ compile_time_parameters:
     - 10.0.2.0/28
 ```
 
-Keys in parameter files are automatically converted to camel case.
+#### SparkleFormation
+
+Compile time parameters can be used for [SparkleFormation](http://www.sparkleformation.io) templates. It conforms and
+allows you to use the [Compile Time Parameters](http://www.sparkleformation.io/docs/sparkle_formation/compile-time-parameters.html) feature.
+
+#### CloudFormation YAML ERB
+
+Compile time parameters can be used to pre-process YAML CloudFormation templates. An example template:
+
+```yaml
+# templates/some_stack_template.yml.erb
+Parameters:
+  VpcCidr:
+    Type: String
+Resources:
+  Vpc:
+    Type: AWS::EC2::VPC
+    Properties:
+      CidrBlock: !Ref VpcCidr
+  # Given the two subnet_cidrs parameters, this creates two resources:
+  # SubnetPrivate0 with a CidrBlock of 10.0.0.0/28, and
+  # SubnetPrivate1 with a CidrBlock of 10.0.2.0/28
+  <% params["SubnetCidrs"].each_with_index do |cidr, index| %>
+  SubnetPrivate<%= index %>:
+    Type: AWS::EC2::Subnet
+    Properties:
+      VpcId: !Ref Vpc
+      AvailabilityZone: ap-southeast-2
+      CidrBlock: <%= cidr %>
+  <% end %>
+```
 
 ## Parameter Resolvers
 
@@ -688,6 +720,7 @@ stacks:
 
 In the cases where you want to bypass the account check, there is the StackMaster flag `--skip-account-check` that can be used.
 
+
 ## Commands
 
 ```bash
@@ -701,15 +734,20 @@ stack_master apply # Create or update all stacks
 stack_master --changed apply # Create or update all stacks that have changed
 stack_master --yes apply [region-or-alias] [stack-name] # Create or update a stack non-interactively (forcing yes)
 stack_master diff [region-or-alias] [stack-name] # Display a stack template and parameter diff
+stack_master drift [region-or-alias] [stack-name] # Detects and displays stack drift using the CloudFormation Drift API
 stack_master delete [region-or-alias] [stack-name] # Delete a stack
 stack_master events [region-or-alias] [stack-name] # Display events for a stack
 stack_master outputs [region-or-alias] [stack-name] # Display outputs for a stack
 stack_master resources [region-or-alias] [stack-name] # Display outputs for a stack
 stack_master status # Displays the status of each stack
 stack_master tidy # Find missing or extra templates or parameter files
+stack_master compile # Print the compiled version of a given stack
+stack_master validate # Validate a template
+stack_master lint # Check the stack definition locally using cfn-lint
+stack_master nag # Check the stack template with cfn_nag
 ```
 
-## Applying updates
+## Applying updates - `stack_master apply`
 
 The apply command does the following:
 
@@ -726,6 +764,18 @@ Demo:
 
 ![Apply Demo](/apply_demo.gif?raw=true)
 
+## Drift Detection - `stack_master drift`
+
+`stack_master drift us-east-1 mystack` uses the CloudFormation APIs to trigger drift detection and display resources
+that have changed outside of the CloudFormation stack. This can happen if a resource has been updated via the console or
+CLI directly rather than via a stack update.
+
+## Diff - `stack_master diff`
+
+`stack_master diff us-east-1 mystack` displays whether the computed parameters or template differ to what was last
+applied in CloudFormation. This can happen if the template or computed parameters have changed in code and the change
+hasn't been applied to this stack.
+
 ## Maintainers
 
 - [Steve Hodgkiss](https://github.com/stevehodgkiss)

data/lib/stack_master.rb

@@ -24,6 +24,7 @@ module StackMaster
   autoload :CLI, 'stack_master/cli'
   autoload :CtrlC, 'stack_master/ctrl_c'
   autoload :Command, 'stack_master/command'
+  autoload :Diff, 'stack_master/diff'
   autoload :VERSION, 'stack_master/version'
   autoload :Stack, 'stack_master/stack'
   autoload :Prompter, 'stack_master/prompter'
@@ -51,11 +52,13 @@ module StackMaster
   require 'stack_master/template_compilers/sparkle_formation'
   require 'stack_master/template_compilers/json'
   require 'stack_master/template_compilers/yaml'
+  require 'stack_master/template_compilers/yaml_erb'
   require 'stack_master/template_compilers/cfndsl'
 
   module Commands
     autoload :TerminalHelper, 'stack_master/commands/terminal_helper'
     autoload :Apply, 'stack_master/commands/apply'
+    autoload :Drift, 'stack_master/commands/drift'
     autoload :Events, 'stack_master/commands/events'
     autoload :Outputs, 'stack_master/commands/outputs'
     autoload :Init, 'stack_master/commands/init'
@@ -68,6 +71,7 @@ module StackMaster
     autoload :Delete, 'stack_master/commands/delete'
     autoload :Status, 'stack_master/commands/status'
     autoload :Tidy, 'stack_master/commands/tidy'
+    autoload :Nag, 'stack_master/commands/nag'
   end
 
   module ParameterResolvers
@@ -198,4 +202,16 @@ module StackMaster
   def stderr=(io)
     @stderr = io
   end
+
+  def colorize(text, color)
+    if colorize?
+      Rainbow(text).color(color)
+    else
+      text
+    end
+  end
+
+  def colorize?
+    ENV.fetch('COLORIZE') { 'true' } == 'true'
+  end
 end

data/lib/stack_master/aws_driver/cloud_formation.rb

@@ -28,7 +28,10 @@ module StackMaster
                           :update_stack,
                           :create_stack,
                           :validate_template,
-                          :describe_stacks
+                          :describe_stacks,
+                          :detect_stack_drift,
+                          :describe_stack_drift_detection_status,
+                          :describe_stack_resource_drifts
 
       private
 

data/lib/stack_master/cli.rb

@@ -146,6 +146,17 @@ module StackMaster
         end
       end
 
+      command :nag do |c|
+        c.syntax = 'stack_master nag [region_or_alias] [stack_name]'
+        c.summary = "Check this stack's template with cfn_nag"
+        c.description = "Runs SAST scan cfn_nag on the template"
+        c.example 'run cfn_nag on stack myapp-vpc with us-east-1 settings', 'stack_master nag us-east-1 myapp-vpc'
+        c.action do |args, options|
+          options.default config: default_config_file
+          execute_stacks_command(StackMaster::Commands::Nag, args, options)
+        end
+      end
+
       command :compile do |c|
         c.syntax = 'stack_master compile [region_or_alias] [stack_name]'
         c.summary = "Print the compiled version of a given stack"
@@ -215,6 +226,18 @@ module StackMaster
         end
       end
 
+      command :drift do |c|
+        c.syntax = 'stack_master drift [region_or_alias] [stack_name]'
+        c.summary = 'Detects and displays stack drift using the CloudFormation Drift API'
+        c.description = 'Detects and displays stack drift'
+        c.option '--timeout SECONDS', Integer, "The number of seconds to wait for drift detection to complete"
+        c.example 'view stack drift for a stack named myapp-vpc in us-east-1', 'stack_master drift us-east-1 myapp-vpc'
+        c.action do |args, options|
+          options.default config: default_config_file, timeout: 120
+          execute_stacks_command(StackMaster::Commands::Drift, args, options)
+        end
+      end
+
       run!
     end
 
@@ -242,6 +265,7 @@ module StackMaster
         stack_definitions = config.filter(region, stack_name)
         if stack_definitions.empty?
           StackMaster.stdout.puts "Could not find stack definition #{stack_name} in region #{region}"
+          show_other_region_candidates(config, stack_name)
           success = false
         end
         stack_definitions = stack_definitions.select do |stack_definition|
@@ -258,6 +282,13 @@ module StackMaster
       @kernel.exit false unless success
     end
 
+    def show_other_region_candidates(config, stack_name)
+      candidates = config.filter(region="", stack_name=stack_name)
+      return if candidates.empty?
+
+      StackMaster.stdout.puts "Stack name #{stack_name} exists in regions: #{candidates.map(&:region).join(', ')}"
+    end
+
     def execute_if_allowed_account(allowed_accounts, &block)
       raise ArgumentError, "Block required to execute this method" unless block_given?
       if running_in_allowed_account?(allowed_accounts)

data/lib/stack_master/commands/drift.rb

@@ -0,0 +1,118 @@
+require 'diffy'
+
+module StackMaster
+  module Commands
+    class Drift
+      include Command
+      include Commander::UI
+
+      DETECTION_COMPLETE_STATES = [
+        'DETECTION_COMPLETE',
+        'DETECTION_FAILED'
+      ]
+
+      def perform
+        detect_stack_drift_result = cf.detect_stack_drift(stack_name: stack_name)
+        drift_results = wait_for_drift_results(detect_stack_drift_result.stack_drift_detection_id)
+
+        puts colorize("Drift Status: #{drift_results.stack_drift_status}", stack_drift_status_color(drift_results.stack_drift_status))
+        return if drift_results.stack_drift_status == 'IN_SYNC'
+
+        failed
+
+        resp = cf.describe_stack_resource_drifts(stack_name: stack_name)
+        resp.stack_resource_drifts.each do |drift|
+          display_drift(drift)
+        end
+      end
+
+      private
+
+      def cf
+        @cf ||= StackMaster.cloud_formation_driver
+      end
+
+      def display_drift(drift)
+        color = drift_color(drift)
+        puts colorize([drift.stack_resource_drift_status,
+                       drift.resource_type,
+                       drift.logical_resource_id,
+                       drift.physical_resource_id].join(' '), color)
+        return unless drift.stack_resource_drift_status == 'MODIFIED'
+
+        unless drift.property_differences.empty?
+          puts colorize('  Property differences:', color)
+        end
+        drift.property_differences.each do |property_difference|
+          puts colorize("  - #{property_difference.difference_type} #{property_difference.property_path}", color)
+        end
+        puts colorize('  Resource diff:', color)
+        display_resource_drift(drift)
+      end
+
+      def display_resource_drift(drift)
+        diff = ::StackMaster::Diff.new(before: prettify_json(drift.expected_properties),
+                        after: prettify_json(drift.actual_properties))
+        diff.display_colorized_diff
+      end
+
+      def prettify_json(string)
+        JSON.pretty_generate(JSON.parse(string)) + "\n"
+      rescue StandardError => e
+        puts "Failed to prettify drifted resource: #{e.message}"
+        string
+      end
+
+      def stack_drift_status_color(stack_drift_status)
+        case stack_drift_status
+        when 'IN_SYNC'
+          :green
+        when 'DRIFTED'
+          :yellow
+        else
+          :blue
+        end
+      end
+
+      def drift_color(drift)
+        case drift.stack_resource_drift_status
+        when 'IN_SYNC'
+          :green
+        when 'MODIFIED'
+          :yellow
+        when 'DELETED'
+          :red
+        else
+          :blue
+        end
+      end
+
+      def wait_for_drift_results(detection_id)
+        resp = nil
+        start_time = Time.now
+        loop do
+          resp = cf.describe_stack_drift_detection_status(stack_drift_detection_id: detection_id)
+          break if DETECTION_COMPLETE_STATES.include?(resp.detection_status)
+
+          elapsed_time = Time.now - start_time
+          if elapsed_time > @options.timeout
+            raise "Timeout waiting for stack drift detection"
+          end
+
+          sleep SLEEP_SECONDS
+        end
+        resp
+      end
+
+      def puts(string)
+        StackMaster.stdout.puts(string)
+      end
+
+      extend Forwardable
+      def_delegators :@stack_definition, :stack_name, :region
+      def_delegators :StackMaster, :colorize
+
+      SLEEP_SECONDS = 1
+    end
+  end
+end

data/lib/stack_master/commands/nag.rb

@@ -0,0 +1,30 @@
+module StackMaster
+  module Commands
+    class Nag
+      include Command
+      include Commander::UI
+
+      def perform
+        rv = Tempfile.open(['stack', "___#{stack_definition.stack_name}.#{proposed_stack.template_format}"]) do |f|
+          f.write(proposed_stack.template_body)
+          f.flush
+          system('cfn_nag', f.path)
+          $?.exitstatus
+        end
+
+        failed!("cfn_nag check failed with exit status #{rv}") if rv > 0
+      end
+
+      private
+
+      def stack_definition
+        @stack_definition ||= @config.find_stack(@region, @stack_name)
+      end
+
+      def proposed_stack
+        @proposed_stack ||= Stack.generate(stack_definition, @config)
+      end
+
+    end
+  end
+end

data/lib/stack_master/config.rb

@@ -92,6 +92,7 @@ module StackMaster
         json: :json,
         yml:  :yaml,
         yaml: :yaml,
+        erb:  :yaml_erb,
       }
     end
 

data/lib/stack_master/diff.rb

@@ -0,0 +1,45 @@
+module StackMaster
+  class Diff
+    def initialize(name: nil, before:, after:, context: 10_000)
+      @name = name
+      @before = before
+      @after = after
+      @context = context
+    end
+
+    def display
+      stdout.print "#{@name} diff: "
+      if diff == ''
+        stdout.puts "No changes"
+      else
+        stdout.puts
+        display_colorized_diff
+      end
+    end
+
+    def display_colorized_diff
+      diff.each_line do |line|
+        if line.start_with?('+')
+          stdout.print colorize(line, :green)
+        elsif line.start_with?('-')
+          stdout.print colorize(line, :red)
+        else
+          stdout.print line
+        end
+      end
+    end
+
+    def different?
+      diff != ''
+    end
+
+    private
+
+    def diff
+      @diff ||= Diffy::Diff.new(@before, @after, context: @context).to_s
+    end
+
+    extend Forwardable
+    def_delegators :StackMaster, :colorize, :stdout
+  end
+end

data/lib/stack_master/sparkle_formation/compile_time/empty_validator.rb

@@ -19,7 +19,7 @@ module StackMaster
 
         def has_invalid_values?
           values = build_values(@definition, @parameter)
-          values.include?(nil) || values.include?('')
+          values.include?(nil)
         end
 
         def create_error

data/lib/stack_master/stack_differ.rb

@@ -10,13 +10,13 @@ module StackMaster
 
     def proposed_template
       return @proposed_stack.template_body unless @proposed_stack.template_format == :json
-      JSON.pretty_generate(JSON.parse(@proposed_stack.template_body))
+      JSON.pretty_generate(JSON.parse(@proposed_stack.template_body)) + "\n"
     end
 
     def current_template
       return '' unless @current_stack
       return @current_stack.template_body unless @current_stack.template_format == :json
-      JSON.pretty_generate(TemplateUtils.template_hash(@current_stack.template_body))
+      JSON.pretty_generate(TemplateUtils.template_hash(@current_stack.template_body)) + "\n"
     end
 
     def current_parameters
@@ -39,24 +39,30 @@ module StackMaster
     end
 
     def body_different?
-      body_diff != ''
+      body_diff.different?
     end
 
     def body_diff
-      @body_diff ||= Diffy::Diff.new(current_template, proposed_template, context: 7, include_diff_info: true).to_s
+      @body_diff ||= Diff.new(name: 'Stack',
+                              before: current_template,
+                              after: proposed_template,
+                              context: 7)
     end
 
     def params_different?
-      params_diff != ''
+      parameters_diff.different?
     end
 
-    def params_diff
-      @param_diff ||= Diffy::Diff.new(current_parameters, proposed_parameters, {}).to_s
+    def parameters_diff
+      @param_diff ||= Diff.new(name: 'Parameters',
+                               before: current_parameters,
+                               after: proposed_parameters)
     end
 
     def output_diff
-      display_diff('Stack', body_diff)
-      display_diff('Parameters', params_diff)
+      body_diff.display
+      parameters_diff.display
+
       unless noecho_keys.empty?
         StackMaster.stdout.puts " * can not tell if NoEcho parameters are different."
       end
@@ -83,38 +89,8 @@ module StackMaster
 
     private
 
-    def display_diff(thing, diff)
-      StackMaster.stdout.print "#{thing} diff: "
-      if diff == ''
-        StackMaster.stdout.puts "No changes"
-      else
-        StackMaster.stdout.puts
-        diff.each_line do |line|
-          if line.start_with?('+')
-            StackMaster.stdout.print colorize(line, :green)
-          elsif line.start_with?('-')
-            StackMaster.stdout.print colorize(line, :red)
-          else
-            StackMaster.stdout.print line
-          end
-        end
-      end
-    end
-
     def sort_params(hash)
       hash.sort.to_h
     end
-
-    def colorize(text, color)
-      if colorize?
-        Rainbow(text).color(color)
-      else
-        text
-      end
-    end
-
-    def colorize?
-      ENV.fetch('COLORIZE') { 'true' } == 'true'
-    end
   end
 end

data/lib/stack_master/template_compilers/yaml_erb.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module StackMaster::TemplateCompilers
+  class YamlErb
+    def self.require_dependencies
+      require 'erubis'
+      require 'yaml'
+    end
+
+    def self.compile(template_dir, template, compile_time_parameters, _compiler_options = {})
+      template_file_path = File.join(template_dir, template)
+      template = Erubis::Eruby.new(File.read(template_file_path))
+      template.filename = template_file_path
+
+      template.result(params: compile_time_parameters)
+    end
+
+    StackMaster::TemplateCompiler.register(:yaml_erb, self)
+  end
+end

data/lib/stack_master/version.rb

@@ -1,3 +1,3 @@
 module StackMaster
-  VERSION = "2.7.0"
+  VERSION = "2.12.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: stack_master
 version: !ruby/object:Gem::Version
-  version: 2.7.0
+  version: 2.12.0
 platform: ruby
 authors:
 - Steve Hodgkiss
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-15 00:00:00.000000000 Z
+date: 2020-10-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -437,6 +437,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: cfn-nag
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.7
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.7
 description: ''
 email:
 - steve@hodgkiss.me
@@ -459,10 +473,12 @@ files:
 - lib/stack_master/commands/compile.rb
 - lib/stack_master/commands/delete.rb
 - lib/stack_master/commands/diff.rb
+- lib/stack_master/commands/drift.rb
 - lib/stack_master/commands/events.rb
 - lib/stack_master/commands/init.rb
 - lib/stack_master/commands/lint.rb
 - lib/stack_master/commands/list_stacks.rb
+- lib/stack_master/commands/nag.rb
 - lib/stack_master/commands/outputs.rb
 - lib/stack_master/commands/resources.rb
 - lib/stack_master/commands/status.rb
@@ -471,6 +487,7 @@ files:
 - lib/stack_master/commands/validate.rb
 - lib/stack_master/config.rb
 - lib/stack_master/ctrl_c.rb
+- lib/stack_master/diff.rb
 - lib/stack_master/identity.rb
 - lib/stack_master/paged_response_accumulator.rb
 - lib/stack_master/parameter_loader.rb
@@ -522,6 +539,7 @@ files:
 - lib/stack_master/template_compilers/json.rb
 - lib/stack_master/template_compilers/sparkle_formation.rb
 - lib/stack_master/template_compilers/yaml.rb
+- lib/stack_master/template_compilers/yaml_erb.rb
 - lib/stack_master/template_utils.rb
 - lib/stack_master/test_driver/cloud_formation.rb
 - lib/stack_master/test_driver/s3.rb
@@ -539,8 +557,8 @@ licenses:
 metadata:
   bug_tracker_uri: https://github.com/envato/stack_master/issues
   changelog_uri: https://github.com/envato/stack_master/blob/master/CHANGELOG.md
-  documentation_uri: https://www.rubydoc.info/gems/stack_master/2.7.0
-  source_code_uri: https://github.com/envato/stack_master/tree/v2.7.0
+  documentation_uri: https://www.rubydoc.info/gems/stack_master/2.12.0
+  source_code_uri: https://github.com/envato/stack_master/tree/v2.12.0
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -556,7 +574,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: StackMaster is a sure-footed way of creating, updating and keeping track