cloudformation-tool 1.2.2 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 15f7ca4dd9a56c71678b984eee089dea7b320bf9e0e7b3d02f72f6d52c3b0182
-  data.tar.gz: 519a9758535c8323f472b512128a96f676eaa611147a0202a09542eb5bee8418
+  metadata.gz: 818f632bb0756f9be2ba922936b2a68e93a12d3af1a10b069e48941dd0615cfd
+  data.tar.gz: 612c5e90b2a95f54435e1ddcbd65a4f833863097c61709a7d9d6895dd041c91e
 SHA512:
-  metadata.gz: f3b0164a2992ee13b4a0278f17fd396f7e4e49718b2c5b7a524894fc1a6002ea070295d3127d646d782b4f3b6d0de85641791ad91932d63b099a1e23b30d79b9
-  data.tar.gz: 41bc98433f5ec0eae331b6733c1f6b26dbdaf25e320b0c57c1763148efd3b62798808f1db612f84c8817210db0555e8caf7c6c709a3960849462a2052fd1baf6
+  metadata.gz: ed6cb8daf2f5990b0a0df87149f9537e29153aacaa65c81a73fee2f2dc05e7d882a99463c22e1fafb00f46757961418df392e730cca805783922ad62db6114ba
+  data.tar.gz: 9d584621509bf30401687ca5c946d9dd45d38a9ad6454b929283099db0ed2ae2aa4ddf93eecf2e123936c4c0a98c605bbe9bfb38c44609c94d4b06e09f626cb1

data/README.md

@@ -103,27 +103,30 @@ multiple sub-templates, this is an error that would cause the tool to abort.
 
 ### Loading user data files
 
-When specifying a user-data block for a `LaunchConfiguration` resource or an `Instance`
-resource, the user-data can be loaded from an external YAML file (only YAML formatted user-data
-is currently supported, sorry) by specifying the `UserData` element as a map with the single
-field `File` that is set with the relative path to the user-data file. The user-data file is
-expected to be a cloud-init configuration file with the default extension `.init`.
-
-Alternatively, the field `FileTemplate` can be used under `UserData` to load an external cloud-init configuration file that includes variable place holders for the
-(CloudFormation intrinsic function Sub)[http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/intrinsic-function-reference-sub.html]. The `FileTemplate` mode supports all
-the features described above as well as it performs the parsing detailed below, except
-compression and S3 offloading - as doing so prevents CloudFormation from performing the
-substitution operation. As a result, if the resulting cloud-init file is larger than 16KB
-you should expect that the template will fail to create the server.
+When specifying a user-data block for a `LaunchConfiguration` resource, `Instance`
+resource, or a `LaunchTemplate` resource, the user-data can be loaded from an external
+YAML file (only YAML formatted user-data is currently supported, sorry) by specifying the
+`UserData` element as a map with the single field `File` that is set with the relative
+path to the user-data file. The user-data file is expected to be a cloud-init configuration
+file with the default extension `.init` (but there really aren't any filename requirements).
+
+Alternatively, the field `FileTemplate` can be used under `UserData` to load an external
+cloud-init configuration file that includes variable place holders for the
+(CloudFormation intrinsic function Sub)[http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/intrinsic-function-reference-sub.html].
+The `FileTemplate` mode supports all the features described above as well as it performs
+the parsing detailed below, except compression and S3 offloading - as doing so prevents
+CloudFormation from performing the substitution operation. As a result, if the resulting
+cloud-init file is larger than 16KB you should expect that the template will fail to create
+the stack.
 
 #### User data file parsing
 
 The reference file will be loaded and parsed as a ("Cloud Config data" file)[http://cloudinit.readthedocs.io/en/latest/topics/format.html#cloud-config-data]
-with the special `write_files` and `write_directories` enhancement (see below). The result is then checked that it
-does not exceed the user-data size limitation. If the file is bigger than can fit in the AWS
-user-data block, it will first be compressed using gzip and if it is still too large, it will
-be uploaded to S3 and the user-data block will be set with a cloud-init download reference to
-the S3 object.
+with the special `write_files` and `write_directories` enhancement (see below). 
+The result is then checked that it does not exceed the user-data size limitation.
+If the file is bigger than can fit in the AWS user-data block, it will first be compressed
+using gzip and if it is still too large, it will be uploaded to S3 and the user-data block
+will be set with a cloud-init download reference to the S3 object.
 
 ##### Enhanced `write_files`
 
@@ -216,6 +219,102 @@ specifying the S3 bucket and object key, either of the following fields may be u
       Role: !GetAtt [ LambdaExecutionRole, Arn ]
 ```
 
+### Nested Stacks Modules
+
+The CloudFormation pre-compiler supports loading local templates as "nested stacks" using the
+CloudFormation `AWS::CloudFormation::Stack` resource type.
+
+Instead of first pre-deploying a template to S3 to be used for a nested stack, use the
+`Template` property (instead of the `TemplateURL` property) to point to a local
+sub-template. The sub-template will be compiled separately and deployed automatically to
+an S3 bucket before deploying the compiled template to CloudFormation.
+
+The `monitor` tool (also used during `create` operation) supports nested stacks by
+automatically detecting nested stack updates in the main stack's event stream and will
+start streaming the nested stack events - this allows the user to more easily locate problems
+with nested stacks.
+
+Currently there's no automatic resolution of references between nested and parent stacks, so
+make sure to set up nested stack parameters for all resources that should be referenced from
+the parent stack.
+
+#### Example
+
+`cloud-formation.yaml`:
+
+```
+AWSTemplateFormatVersion: "2010-09-09"
+Description: "CloudFormation template with nested stacks"
+Parameters:
+  DomainName:
+    Description: "The DNS domain name for the system"
+    Type: String
+    Default: example.com
+  AMI:
+    Description: "The AMI ID for the image to deploy"
+    Type: String
+    Default: ami-af4333cf
+
+Resources:
+  VPC:
+    Type: AWS::EC2::VPC
+    Properties:
+      CidrBlock: 172.20.0.0/16
+      EnableDnsSupport: true
+      EnableDnsHostnames: true
+  SecurityGroupExample:
+    Type: AWS::EC2::SecurityGroup
+    Properties:
+      VpcId: !Ref VPC 
+      GroupDescription: example security group
+      SecurityGroupIngress:
+        - { IpProtocol: icmp, CidrIp: 0.0.0.0/0, FromPort: -1, ToPort: -1 }
+        - { IpProtocol: tcp, CidrIp: 0.0.0.0/0, FromPort: 22, ToPort: 22 }
+  ServiceStack:
+    Type: AWS::CloudFormation::Stack
+    Properties:
+      Template: service.yaml
+      Parameters:
+        DomainName: !Ref DomainName
+        AMI: !Ref AMI
+        VPC: !Ref VPC
+```
+
+`service.yaml`:
+
+```
+AWSTemplateFormatVersion: "2010-09-09"
+Description: "Service nested stack"
+Parameters:
+  DomainName:
+    Description: "The DNS domain name for the system"
+    Type: String
+  AMI:
+    Description: "The AMI ID for the image to deploy"
+    Type: String
+  VPC:
+    Description: "The VPC into which to deploy the service"
+    Type: String
+
+Resources:
+  Subnet:
+    Type: AWS::EC2::Subnet
+    Properties:
+      AvailabilityZone: !Select [ 0, !GetAZs { Ref: "AWS::Region" } ]
+      CidrBlock: 172.20.0.0/24
+      MapPublicIpOnLaunch: true
+      VpcId: !Ref VPC
+  Ec2Instance: 
+    Type: AWS::EC2::Instance
+    Properties: 
+      ImageId: !Ref AMI
+      KeyName: "secret" 
+      NetworkInterfaces: 
+        - AssociatePublicIpAddress: "true"
+          DeviceIndex: "0"
+          SubnetId: !Ref Subnet 
+```
+
 ## Caching
 
 Some resource compilation may require uploading to S3, such as Lambda code or cloud-init setup
@@ -248,7 +347,12 @@ The following commands are supported:
  - `status` - Check if the names stack exists or not
  - `delete` - Delete the specified stack. After issuing the delete command, the tool will
    immediately start `monitor` mode until the operation has completed. 
- - `servers` - List EC2 instances created and managed by this stack.
+ - `servers` - List EC2 instances created and managed by this stack, per autoscaling group, including servers in nested stacks.
+ - `groups` - list autoscaling groups managed by the stack, including groups in nested stacks.
+ - `recycle` - recycle servers in an autoscaling group in a stack by scaling the group up and down.
+ - `scale` - set the scale of an autoscaling group managed by a stack to a specific desired value.
+ - `invalidate` - send an invalidation request to a CloudFront distribution managed by a stack.
+ - `output` - retrieve output values from a stack.
 
 Please see the specific help for each command by running `cftool <command> --help` for
 more details and specific options.
@@ -260,3 +364,100 @@ The AWS region to be used can be select by specifying top level option (i.e. bef
 ### Credentials Selection
 
 The tool will use the standard AWS credentials selection process, except when you want to use AWS CLI configured credential profiles, you may select to use a profile other than "default" by specifying the top level option (i.e. before the command name) `-p <profile>`, by providing the standard environment variable `AWS_DEFAULT_PROFILE` or by having a file called `.awsprofile` - whose content is the name of a valid AWS REGION - in a parent directory (at any level up to the root directory).
+
+## Library API
+
+The cloudformatin tool can also be consumed as a library by other applications - for example an application that needs to perform high-level business-logic oriented
+operations for a specific application deployed in a stack, using the cloudformation tool abstraction of CloudFormation templates and stacks.
+
+### Usage as a library
+
+To use the cloudformatin tool as a library, require `cloud_formation_tool`.
+
+### CloudFormation templates
+
+The cloudformation pre-compiler can be used to manipulate pre-compiled templates.
+
+To access the pre-compiler, initialize a `CloudFormationTool::CloudFormation` with the path to the local template resource (either a file or a directory that can be
+parsed by the pre-compiler).
+
+The initial template resource will be loaded but will not be fully parsed - and included elements will not be read - until the `compile` method is called.
+
+The following method calls are available on the `CloudFormation` instance:
+
+#### `compile(parameters = nil)`
+
+Pre-compiles the template, with the provided parameter `Hash`, if provided. Returns a `Hash` repsenting the compiled template.
+
+#### `to_yaml`
+
+Pre-compiles the template and returns a YAML rendering of the CloudFormation template, suitable for deploying to AWS CloudFormation.
+
+#### `each`
+
+Yields a tuple for each defined template parameter, that includes the parameter's name and its default value (if set, `nil` otherwise).
+
+### CloudFormation stacks
+
+The cloudformation tool's abstraction of a CloudFormation stack can be used to manipulate stack resouces, such as autoscaling groups or instances in a stack context.
+
+To access the stack API, initialize a `CloudFormationTool::CloudFormation::Stack` with the name of the stack. You can then access the following methods:
+
+#### `exist?`
+
+Check if a stack exists.
+
+#### `create(template, params = {})`
+
+Create or update a stack by deploying the specified template. The template can be any local file or directory resource that can be parsed by the cloudformation pre-compiler.
+
+#### `delete`
+
+Deletes the stack
+
+#### `stack_id`
+
+Return the AWS CloudFormation stack identifier for the stack, which is the ARN of the stack.
+
+#### `output`
+
+Returns the output values of the stack
+
+#### `resources`
+
+Return a list of resources in the stack and all of its nested stacks
+
+#### `asgroups`
+
+Return a list of autoscaling groups in the stack and all of its nested stacks. The returned values are AWS SDK CloudFormation resources, extended with a set of methods
+to help manage autoscaling groups:
+
+##### `group`
+
+Returns the AWS SDK `Aws::AutoScaling::AutoScalingGroup` object for the autoscaling group.
+
+#### `cdns`
+
+Return a list of CloudFront CDN distributions in the stack and all of its nested stacks. The returnd values are AWS SDK CloudFormation resources, extended with a set of
+methods to help manage CloudFront distributions:
+
+##### `distribution`
+
+Returns the AWS SDK `Aws::CloudFront::Types::Distribution` object for the CloudFront distribution.
+
+##### `domain_names`
+
+Returns the comma delimited list of the distribution aliases domain names
+
+##### `invalidate(path)`
+
+Creates a new invalidation in the CloudFront distribution with the specified path expression
+
+#### `each`
+
+Yields CloudFormation stack events, in the order they were created. Subsequent calls to `each` will not repeat events previously yielded and will only yield additional
+events created since the last call to `each`.
+
+#### `see_event`
+
+Mark all events since the last call to `each` (or from stack creation, if `each` was not previously called) as "seen" so they will not be yielded in future calls to `each`.

data/lib/cloud_formation_tool/cli/main.rb

@@ -31,7 +31,7 @@ module CloudFormationTool
       
       option [ "-d", "--debug" ], :flag, "Enable debug logging" do
         logger.level = Logger::Severity::DEBUG
-        logger.formatter = logger.default_formatter
+        logger.formatter = nil
       end
       
       option [ "-q", "--quiet" ], :flag, "Enable debug logging" do

data/lib/cloud_formation_tool/cloud_formation.rb

@@ -23,7 +23,7 @@ module CloudFormationTool
       begin
         text = File.read(@path)
         # remove comments because white space seen between comments can seriously psych Psych
-        text.gsub!(/^#.*\n/s,'')
+        text.gsub!(/^#.*\n/,'')
         text = fixShorthand(text)
         @data = YAML.load(text).to_h
       rescue Psych::SyntaxError => e
@@ -167,7 +167,7 @@ module CloudFormationTool
         restype = data['Type'] if restype.nil? and data.key?('Type')
         data.inject({}) do |dict, (key, val)|
           dict[key] = case restype
-            when 'AWS::AutoScaling::LaunchConfiguration'
+            when 'AWS::AutoScaling::LaunchConfiguration', 'AWS::EC2::LaunchTemplate'
               if (key == "UserData") and (val["File"]) 
                 # Support LaunchConfiguration UserData from file
                 CloudInit.new("#{@basedir}/#{val["File"]}").to_base64
@@ -183,6 +183,12 @@ module CloudFormationTool
               else
                 load_files(val, restype)
               end
+            when 'AWS::CloudFormation::Stack'
+              if key == 'Properties' and val.key?('Template')
+                NestedStack.new(val, self).to_cloudformation
+              else
+                load_files(val, restype)
+              end
             else
               load_files(val, restype)
           end

data/lib/cloud_formation_tool/cloud_formation/nested_stack.rb

@@ -0,0 +1,25 @@
+module CloudFormationTool
+  class CloudFormation
+    
+    class NestedStack
+      include Storable
+      
+      def initialize(props, tpl)
+        @tpl = tpl
+        @data = props
+        if props.key?('Template')
+          path = props['Template']
+          path = if path.start_with? "/" then path else "#{@tpl.basedir}/#{path}" end
+          @content = CloudFormation.new(path).to_yaml
+          @data['TemplateURL'] = upload(make_filename('yaml'), @content, mime_type: 'text/yaml', gzip: false)
+          @data.delete('Template')
+        end
+      end
+      
+      def to_cloudformation
+        @data
+      end
+
+    end
+  end
+end

data/lib/cloud_formation_tool/cloud_formation/stack.rb

@@ -15,6 +15,7 @@ module CloudFormationTool
         @name = name
         @seenev = Set.new
         @watch_timeouts = 0
+        @nested_stacks = Hash[]
       end
       
       def delete
@@ -95,7 +96,14 @@ module CloudFormationTool
       def resources
         begin
           awscf.list_stack_resources(stack_name: @name).each do |resp|
-            resp.stack_resource_summaries.each { |res| yield res }
+            resp.stack_resource_summaries.each do |res|
+              yield res
+              if res.resource_type == 'AWS::CloudFormation::Stack'
+                Stack.new(res.physical_resource_id).resources do |nested_res|
+                  yield nested_res
+                end
+              end
+            end
           end
         rescue Aws::CloudFormation::Errors::ValidationError => e
           raise CloudFormationTool::Errors::AppError, "Failed to get resources: #{e.message}"
@@ -133,6 +141,7 @@ module CloudFormationTool
       end
       
       def monitor(start_time = nil)
+        @nested_stacks = Hash[]
         done = false
         begin
           until done
@@ -140,26 +149,65 @@ module CloudFormationTool
               next if @seenev.add?(ev.event_id).nil?
               text = "#{ev.timestamp.strftime "%Y-%m-%d %H:%M:%S"}| " + %w(
                 resource_type:40
-                logical_resource_id:38
+                logical_resource_id:42
                 resource_status
               ).collect { |field|
                 (name,size) = field.split(":")
                 size ||= 1
-                ev.send(name.to_sym).ljust(size.to_i, ' ')
+                (if name == 'logical_resource_id' and ev.stack_name != self.name
+                  logical_nested_stack_name(ev.stack_name) + "|"
+                else
+                  ''
+                end + ev.send(name.to_sym)).ljust(size.to_i, ' ')
               }.join("  ")
               text += " " + ev.resource_status_reason if ev.resource_status =~ /_FAILED/
               if start_time.nil? or start_time < ev.timestamp
                 puts text
               end
-              done = (ev.resource_type == "AWS::CloudFormation::Stack" and ev.resource_status =~ /(_COMPLETE|_FAILED)$/)
+              check_nested_stack(ev)
+              done = is_final_event(ev)
             end
+            sleep 1
           end
         rescue CloudFormationTool::Errors::StackDoesNotExistError => e
           puts "Stack #{name} does not exist"
         end
       end
       
+      def logical_nested_stack_name(phys_name)
+        @nested_stacks[phys_name] || 'unknown'
+      end
+      
+      def nested_stack_name(ev)
+        ev.physical_resource_id.split('/')[1]
+      end
+      
+      def check_nested_stack(ev)
+        return unless ev.resource_type == "AWS::CloudFormation::Stack" and
+          ev.logical_resource_id != self.name # not nested stack
+        return if @nested_stacks.has_key? ev.logical_resource_id # seeing the first or last nested stack event - ignoring
+        @nested_stacks[nested_stack_name(ev)] = ev.logical_resource_id
+      end
+      
+      def is_final_event(ev)
+        ev.resource_type == "AWS::CloudFormation::Stack" and
+        ev.resource_status =~ /(_COMPLETE|_FAILED)$/ and
+        ev.logical_resource_id == self.name
+      end
+      
+      def tracked_stacks
+        [ self.name ] + @nested_stacks.keys.compact
+      end
+      
       def each
+        tracked_stacks.each do |name|
+          events_for(name) do |ev|
+            yield ev
+          end
+        end
+      end
+      
+      def events_for(stack_name)
         token = nil
         sleep(if @_last_poll_time.nil?
           0
@@ -172,7 +220,7 @@ module CloudFormationTool
             end
           end)
         begin
-          resp = awscf.describe_stack_events stack_name: name, next_token: token
+          resp = awscf.describe_stack_events stack_name: stack_name, next_token: token
           @watch_timeouts = 0
           resp.stack_events.each do |ev|
             yield ev
@@ -188,7 +236,11 @@ module CloudFormationTool
           end
         rescue Aws::CloudFormation::Errors::ValidationError => e
           if e.message =~ /does not exist/
-            raise CloudFormationTool::Errors::StackDoesNotExistError, "Stack does not exist"
+            if stack_name == self.name
+              raise CloudFormationTool::Errors::StackDoesNotExistError, "Stack does not exist"
+            end
+            # ignore "does not exist" errors on nested stacks - we may try to poll them before
+            # they actually exist. We'll just try later
           else
             raise e
           end

data/lib/cloud_formation_tool/cloud_init.rb

@@ -45,7 +45,7 @@ module CloudFormationTool
     def encode(allow_gzip = true)
       yamlout = compile
       usegzip = false
-      if allow_gzip and yamlout.size > 16384 # max AWS EC2 user data size - try compressing it
+      if allow_gzip and yamlout.size > 16000 # max AWS EC2 user data size - try compressing it
         yamlout = Zlib::Deflate.new(nil, 31).deflate(yamlout, Zlib::FINISH) # 31 is the magic word to have deflate create a gzip compatible header
         usegzip = true
       end

data/lib/cloud_formation_tool/version.rb

@@ -1,3 +1,3 @@
 module CloudFormationTool
-  VERSION = '1.2.2'
+  VERSION = '1.4.0'
 end

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: cloudformation-tool
 version: !ruby/object:Gem::Version
-  version: 1.2.2
+  version: 1.4.0
 platform: ruby
 authors:
 - Oded Arbel
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-12-23 00:00:00.000000000 Z
+date: 2021-01-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '12'
+        version: 12.3.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '12'
+        version: 12.3.3
 - !ruby/object:Gem::Dependency
   name: clamp
   requirement: !ruby/object:Gem::Requirement
@@ -168,6 +168,7 @@ files:
 - lib/cloud_formation_tool/cloud_formation.rb
 - lib/cloud_formation_tool/cloud_formation/cloud_front_distribution.rb
 - lib/cloud_formation_tool/cloud_formation/lambda_code.rb
+- lib/cloud_formation_tool/cloud_formation/nested_stack.rb
 - lib/cloud_formation_tool/cloud_formation/stack.rb
 - lib/cloud_formation_tool/cloud_init.rb
 - lib/cloud_formation_tool/errors.rb
@@ -192,7 +193,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.1
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: A pre-compiler tool for CloudFormation YAML templates