stack_master 0.6.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 462256c45f5be50e9afe763ac0bd7dc8a466ce98
-  data.tar.gz: d2feec3d48f34397eef2e26223c15e9745fd5bcb
+  metadata.gz: 24f9543391ced08e41349c5cd75aaecacce6aab2
+  data.tar.gz: 0130dc01c3bfacea757155caf6896aef4daaebaf
 SHA512:
-  metadata.gz: c78d5094529357d8e17581d46092a7b8f5ff155cf8ffb2081be549fb081c331bf96d97f35e6cb6f6de208fd6edb79baba2f2766b459667a87acf05de0c08525b
-  data.tar.gz: ddc6dae60758049e6b01a538254e29e14c048d5c82fc63ef3afa59c897f1e7575531c124db1e2fc702aae803ead82a7e3b28d85aabd14f1ca7af0e94cbfa1f41
+  metadata.gz: 95bf919587f2d04956c97ef82caa0f9d79747b2272e1c97075549950a5d5606dbf084cfba5ef65a4e61935a4d3c11fd13c06183f02e9305f55e8f98353ded7be
+  data.tar.gz: d5523800109d4f3ac1a5a04a4f2f3246ad51d2014b76234c2b35c54f18e39b24a8b7fe4e5b9696682d1b299f8bf5b969961b0d49355c54381d2cc247e7cde26d

data/README.md

@@ -95,6 +95,28 @@ stacks:
         purpose: vpc
 ```
 
+## S3
+
+StackMaster can optionally use S3 to store the templates before creating a stack.
+This requires to configure an S3 bucket in stack_master.yml:
+
+```yaml
+stack_defaults:
+  s3:
+    bucket: my_bucket_name
+    prefix: cfn_templates/my-awesome-app
+    region: us-west-2
+```
+
+Additional files can be configured to be uploaded to S3 alongside the templates:
+```yaml
+stacks:
+  production:
+    myapp-vpc:
+      template: myapp_vpc.rb
+      files:
+        - userdata.sh
+```
 ## Directories
 
 - `templates` - CloudFormation, SparkleFormation or CfnDsl templates.

data/features/apply.feature

@@ -90,7 +90,7 @@ Feature: Apply command
       | Parameters diff: |
       | KeyName: my-key  |
       | aborted          |
-    And the output should not match /2020-10-29 00:00:00 \+[0-9]{4} myapp-vpc AWS::CloudFormation::Stack CREATE_COMPLETE/ 
+    And the output should not match /2020-10-29 00:00:00 \+[0-9]{4} myapp-vpc AWS::CloudFormation::Stack CREATE_COMPLETE/
     Then the exit status should be 0
 
   Scenario: Run apply with region only and create 2 stacks

data/features/apply_with_s3.feature

@@ -0,0 +1,106 @@
+Feature: Apply command
+
+  Background:
+    Given a file named "stack_master.yml" with:
+      """
+      stack_defaults:
+        s3:
+          bucket: my-bucket
+          region: us-east-1
+          prefix: cfn_templates/my-app
+      stacks:
+        us_east_1:
+          myapp_vpc:
+            template: myapp_vpc.rb
+            files:
+              - user_data.sh
+      """
+    And a directory named "parameters"
+    And a file named "parameters/myapp_vpc.yml" with:
+      """
+      KeyName: my-key
+      """
+    And a file named "parameters/myapp_web.yml" with:
+      """
+      VpcId: vpc-blah
+      """
+    And a directory named "templates"
+    And a file named "files/user_data.sh" with:
+      """
+      #!/bin/bash
+      echo "HI"
+      """
+    And a file named "templates/myapp_vpc.rb" with:
+      """
+      SparkleFormation.new(:myapp_vpc) do
+        description "Test template"
+        set!('AWSTemplateFormatVersion', '2010-09-09')
+
+        parameters.key_name do
+          description 'Key name'
+          type 'String'
+        end
+
+        resources.vpc do
+          type 'AWS::EC2::VPC'
+          properties do
+            cidr_block '10.200.0.0/16'
+          end
+        end
+
+        outputs do
+          vpc_id do
+            description 'A VPC ID'
+            value ref!(:vpc)
+          end
+        end
+      end
+      """
+
+  Scenario: Run apply and create a new stack
+    Given I stub the following stack events:
+      | stack_id | event_id | stack_name | logical_resource_id | resource_status | resource_type              | timestamp           |
+      | 1        | 1        | myapp-vpc  | TestSg              | CREATE_COMPLETE | AWS::EC2::SecurityGroup    | 2020-10-29 00:00:00 |
+      | 1        | 1        | myapp-vpc  | myapp-vpc           | CREATE_COMPLETE | AWS::CloudFormation::Stack | 2020-10-29 00:00:00 |
+    When I run `stack_master apply us-east-1 myapp-vpc --trace`
+    And the output should contain all of these lines:
+      | Stack diff:                                                                    |
+      | +    "Vpc": {                                                                  |
+      | Parameters diff:                                                               |
+      | KeyName: my-key                                                                |
+    And the output should match /2020-10-29 00:00:00 \+[0-9]{4} myapp-vpc AWS::CloudFormation::Stack CREATE_COMPLETE/
+    And an S3 file in bucket "my-bucket" with key "cfn_templates/my-app/myapp_vpc.json" exists with content:
+      """
+      {
+        "Description": "Test template",
+        "AWSTemplateFormatVersion": "2010-09-09",
+        "Parameters": {
+          "KeyName": {
+            "Description": "Key name",
+            "Type": "String"
+          }
+        },
+        "Resources": {
+          "Vpc": {
+            "Type": "AWS::EC2::VPC",
+            "Properties": {
+              "CidrBlock": "10.200.0.0/16"
+            }
+          }
+        },
+        "Outputs": {
+          "VpcId": {
+            "Description": "A VPC ID",
+            "Value": {
+              "Ref": "Vpc"
+            }
+          }
+        }
+      }
+      """
+    And an S3 file in bucket "my-bucket" with key "cfn_templates/my-app/user_data.sh" exists with content:
+      """
+      #!/bin/bash
+      echo "HI"
+      """
+    Then the exit status should be 0

data/features/step_definitions/stack_steps.rb

@@ -64,3 +64,8 @@ end
 Given(/^I stub CloudFormation validate calls to fail validation with message "([^"]*)"$/) do |message|
   allow(StackMaster.cloud_formation_driver).to receive(:validate_template).and_raise(Aws::CloudFormation::Errors::ValidationError.new('', message))
 end
+
+When(/^an S3 file in bucket "([^"]*)" with key "([^"]*)" exists with content:$/) do |bucket, key, body|
+  file = StackMaster.s3_driver.find_file(bucket: bucket, object_key: key)
+  expect(file).to eq body
+end

data/features/support/env.rb

@@ -12,4 +12,5 @@ end
 
 Before do
   StackMaster.cloud_formation_driver.reset
+  StackMaster.s3_driver.reset
 end

data/lib/stack_master.rb

@@ -1,69 +1,82 @@
 require "commander"
 require "yaml"
-require "virtus"
 require "aws-sdk"
-require "diffy"
 require "colorize"
-require "table_print"
 require 'active_support/core_ext/string'
-require "erb"
-require 'sparkle_formation'
-require 'dotgpg'
-require 'ruby-progressbar'
-
-require "stack_master/ctrl_c"
-require "stack_master/sparkle_formation/user_data_file"
-require "stack_master/command"
-require "stack_master/version"
-require "stack_master/stack"
-require "stack_master/prompter"
-require "stack_master/aws_driver/cloud_formation"
-require "stack_master/test_driver/cloud_formation"
-require "stack_master/stack_events/fetcher"
-require "stack_master/stack_events/presenter"
-require "stack_master/stack_events/streamer"
-require "stack_master/stack_states"
-require "stack_master/stack_status"
-require "stack_master/sns_topic_finder"
-require "stack_master/security_group_finder"
-require "stack_master/parameter_loader"
-require "stack_master/parameter_resolver"
-require "stack_master/resolver_array"
-require "stack_master/parameter_resolvers/ami_finder"
-require "stack_master/parameter_resolvers/stack_output"
-require "stack_master/parameter_resolvers/secret"
-require "stack_master/parameter_resolvers/sns_topic_name"
-require "stack_master/parameter_resolvers/security_group"
-require "stack_master/parameter_resolvers/latest_ami_by_tags"
-require "stack_master/parameter_resolvers/latest_ami"
-require "stack_master/utils"
-require "stack_master/config"
-require "stack_master/paged_response_accumulator"
-require "stack_master/stack_definition"
-require "stack_master/template_compiler"
-require "stack_master/template_compilers/sparkle_formation"
-require "stack_master/template_compilers/json"
-require "stack_master/template_compilers/yaml"
-require "stack_master/template_compilers/cfndsl"
-require "stack_master/commands/terminal_helper"
-require "stack_master/commands/apply"
-require "stack_master/change_set"
-require "stack_master/commands/events"
-require "stack_master/commands/outputs"
-require "stack_master/commands/init"
-require "stack_master/commands/diff"
-require "stack_master/commands/list_stacks"
-require "stack_master/commands/validate"
-require "stack_master/commands/resources"
-require "stack_master/commands/delete"
-require "stack_master/commands/status"
-require "stack_master/stack_differ"
-require "stack_master/validator"
-require "stack_master/cli"
 
 module StackMaster
   extend self
 
+  autoload :Initializable, 'stack_master/utils/initializable'
+  autoload :ChangeSet, 'stack_master/change_set'
+  autoload :CLI, 'stack_master/cli'
+  autoload :CtrlC, 'stack_master/ctrl_c'
+  autoload :Command, 'stack_master/command'
+  autoload :Version, 'stack_master/version'
+  autoload :Stack, 'stack_master/stack'
+  autoload :Prompter, 'stack_master/prompter'
+  autoload :StackStates, 'stack_master/stack_states'
+  autoload :StackStatus, 'stack_master/stack_status'
+  autoload :SnsTopicFinder, 'stack_master/sns_topic_finder'
+  autoload :SecurityGroupFinder, 'stack_master/security_group_finder'
+  autoload :ParameterLoader, 'stack_master/parameter_loader'
+  autoload :ParameterResolver, 'stack_master/parameter_resolver'
+  autoload :ResolverArray, 'stack_master/resolver_array'
+  autoload :Resolver, 'stack_master/resolver_array'
+  autoload :Utils, 'stack_master/utils'
+  autoload :Config, 'stack_master/config'
+  autoload :PagedResponseAccumulator, 'stack_master/paged_response_accumulator'
+  autoload :StackDefinition, 'stack_master/stack_definition'
+  autoload :TemplateCompiler, 'stack_master/template_compiler'
+
+  autoload :StackDiffer, 'stack_master/stack_differ'
+  autoload :Validator, 'stack_master/validator'
+
+  require 'stack_master/template_compilers/sparkle_formation'
+  require 'stack_master/template_compilers/json'
+  require 'stack_master/template_compilers/yaml'
+  require 'stack_master/template_compilers/cfndsl'
+
+  module Commands
+    autoload :TerminalHelper, 'stack_master/commands/terminal_helper'
+    autoload :Apply, 'stack_master/commands/apply'
+    autoload :Events, 'stack_master/commands/events'
+    autoload :Outputs, 'stack_master/commands/outputs'
+    autoload :Init, 'stack_master/commands/init'
+    autoload :Diff, 'stack_master/commands/diff'
+    autoload :ListStacks, 'stack_master/commands/list_stacks'
+    autoload :Validate, 'stack_master/commands/validate'
+    autoload :Resources, 'stack_master/commands/resources'
+    autoload :Delete, 'stack_master/commands/delete'
+    autoload :Status, 'stack_master/commands/status'
+  end
+
+  module ParameterResolvers
+    autoload :AmiFinder, 'stack_master/parameter_resolvers/ami_finder'
+    autoload :StackOutput, 'stack_master/parameter_resolvers/stack_output'
+    autoload :Secret, 'stack_master/parameter_resolvers/secret'
+    autoload :SnsTopicName, 'stack_master/parameter_resolvers/sns_topic_name'
+    autoload :SecurityGroup, 'stack_master/parameter_resolvers/security_group'
+    autoload :LatestAmiByTags, 'stack_master/parameter_resolvers/latest_ami_by_tags'
+    autoload :LatestAmi, 'stack_master/parameter_resolvers/latest_ami'
+  end
+
+  module AwsDriver
+    autoload :CloudFormation, 'stack_master/aws_driver/cloud_formation'
+    autoload :S3, 'stack_master/aws_driver/s3'
+  end
+
+  module TestDriver
+    autoload :CloudFormation, 'stack_master/test_driver/cloud_formation'
+    autoload :S3, 'stack_master/test_driver/s3'
+  end
+
+  module StackEvents
+    autoload :Fetcher, 'stack_master/stack_events/fetcher'
+    autoload :Presenter, 'stack_master/stack_events/presenter'
+    autoload :Streamer, 'stack_master/stack_events/streamer'
+  end
+
   def interactive?
     !non_interactive?
   end
@@ -106,6 +119,14 @@ module StackMaster
     @cloud_formation_driver = value
   end
 
+  def s3_driver
+    @s3_driver ||= AwsDriver::S3.new
+  end
+
+  def s3_driver=(value)
+    @s3_driver = value
+  end
+
   def stdout
     @stdout || $stdout
   end
@@ -126,4 +147,3 @@ module StackMaster
     @stderr = io
   end
 end
-

data/lib/stack_master/aws_driver/s3.rb

@@ -0,0 +1,66 @@
+require 'digest/md5'
+
+module StackMaster
+  module AwsDriver
+    class S3ConfigurationError < StandardError; end
+
+    class S3
+      def set_region(region)
+        @region = region
+        @s3 = nil
+      end
+
+      def upload_files(bucket: nil, prefix: nil, region: nil, files: {})
+        raise StackMaster::AwsDriver::S3ConfigurationError, 'A bucket must be specified in order to use S3' unless bucket
+
+        return if files.empty?
+
+        s3 = new_s3_client(region: region)
+
+        current_objects = s3.list_objects(
+          prefix: prefix,
+          bucket: bucket
+        ).map(&:contents).flatten.inject({}){|h,obj|
+          h.merge(obj.key => obj)
+        }
+
+        StackMaster.stdout.puts "Uploading files to S3:"
+
+        files.each do |template, file|
+          body = file.fetch(:body)
+          path = file.fetch(:path)
+          object_key = template.dup
+          object_key.prepend("#{prefix}/") if prefix
+          compiled_template_md5 = Digest::MD5.hexdigest(body).to_s
+          s3_md5 = current_objects[object_key] ? current_objects[object_key].etag.gsub("\"", '') : nil
+
+          next if compiled_template_md5 == s3_md5
+          s3_uri = "s3://#{bucket}/#{object_key}"
+          StackMaster.stdout.print "- #{File.basename(path)} => #{s3_uri} "
+
+          s3.put_object(
+            bucket: bucket,
+            key: object_key,
+            body: body,
+            metadata: { md5: compiled_template_md5 }
+          )
+          StackMaster.stdout.puts "done."
+        end
+      end
+
+      def url(bucket:, prefix:, region:, template:)
+        if region == 'us-east-1'
+          ["https://s3.amazonaws.com", bucket, prefix, template].compact.join('/')
+        else
+          ["https://s3-#{region}.amazonaws.com", bucket, prefix, template].compact.join('/')
+        end
+      end
+
+      private
+
+      def new_s3_client(region: nil)
+        Aws::S3::Client.new(region: region || @region)
+      end
+    end
+  end
+end

data/lib/stack_master/cli.rb

@@ -1,4 +1,5 @@
 require 'commander'
+require 'table_print'
 
 module StackMaster
   class CLI

data/lib/stack_master/commands/apply.rb

@@ -4,10 +4,11 @@ module StackMaster
       include Command
       include Commander::UI
       include StackMaster::Prompter
-      TEMPLATE_TOO_LARGE_ERROR_MESSAGE = 'The (space compressed) stack is larger than the limit set by AWS. See http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/cloudformation-limits.html'
+      TEMPLATE_TOO_LARGE_ERROR_MESSAGE = 'The (space compressed) stack is larger than the limit set by AWS. See http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/cloudformation-limits.html'.freeze
 
       def initialize(config, stack_definition, options = {})
         @config = config
+        @s3_config = stack_definition.s3
         @stack_definition = stack_definition
         @from_time = Time.now
       end
@@ -26,6 +27,10 @@ module StackMaster
         @cf ||= StackMaster.cloud_formation_driver
       end
 
+      def s3
+        @s3 ||= StackMaster.s3_driver
+      end
+
       def stack
         @stack ||= Stack.find(region, stack_name)
       end
@@ -38,6 +43,10 @@ module StackMaster
         !stack.nil?
       end
 
+      def use_s3?
+        !@s3_config.empty?
+      end
+
       def diff_stacks
         StackDiffer.new(proposed_stack, stack).output_diff
       end
@@ -51,9 +60,10 @@ module StackMaster
       end
 
       def create_stack
-        unless ask?("Create stack (y/n)? ")
+        unless ask?('Create stack (y/n)? ')
           failed!("Stack creation aborted")
         end
+        upload_files
         cf.create_stack(stack_options.merge(tags: proposed_stack.aws_tags))
       end
 
@@ -73,17 +83,54 @@ module StackMaster
           ChangeSet.delete(@change_set.id)
           halt! "Stack update aborted"
         end
+        upload_files
         execute_change_set
       end
 
+      def upload_files
+        return unless use_s3?
+        s3.upload_files(s3_options)
+      end
+
+      def template_method
+        use_s3? ? :template_url : :template_body
+      end
+
+      def template_value
+        if use_s3?
+          s3.url(bucket: @s3_config['bucket'], prefix: @s3_config['prefix'], region: @s3_config['region'], template: @stack_definition.s3_template_file_name)
+        else
+          proposed_stack.maybe_compressed_template_body
+        end
+      end
+
+      def files_to_upload
+        return {} unless use_s3?
+        @stack_definition.s3_files.tap do |files|
+          files[@stack_definition.s3_template_file_name] = {
+            path: @stack_definition.template_file_path,
+            body: proposed_stack.maybe_compressed_template_body
+          }
+        end
+      end
+
       def stack_options
         {
           stack_name: stack_name,
-          template_body: proposed_stack.maybe_compressed_template_body,
           parameters: proposed_stack.aws_parameters,
           capabilities: ['CAPABILITY_IAM'],
           notification_arns: proposed_stack.notification_arns,
-          stack_policy_body: proposed_stack.stack_policy_body
+          stack_policy_body: proposed_stack.stack_policy_body,
+          template_method => template_value
+        }
+      end
+
+      def s3_options
+        {
+          bucket: @s3_config['bucket'],
+          prefix: @s3_config['prefix'],
+          region: @s3_config['region'],
+          files: files_to_upload
         }
       end
 
@@ -110,7 +157,7 @@ module StackMaster
       end
 
       def ensure_valid_template_body_size!
-        if proposed_stack.too_big?
+        if proposed_stack.too_big?(use_s3?)
           failed! TEMPLATE_TOO_LARGE_ERROR_MESSAGE
         end
       end