cloudformation-tool 1.2.1 → 1.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 06e60f20bd426c7bc9a5a4155b245e9c821a04efc9d160f38698597c65540786
-  data.tar.gz: 42c4c4591c884a273cc5e5afc4987d96f6455faa1fe2e0bae09fadff58112072
+  metadata.gz: 5e68236a0ceae7d8423b08ad878d8800398f6f3d4035883961369db5cce9e69a
+  data.tar.gz: 2f0d5633efd0cd1a20caa428135e1e81f483855dfa77dedaabc9d3b8657a2d01
 SHA512:
-  metadata.gz: 5965d055430f3015261c79585a42fc7fe4f45bc42c982a243b3aa161a883dee49066db543c7a793da60050c07181bc33ca5b3cc505d68b21af6477f9611ae720
-  data.tar.gz: 57486ffee6db6beb3ac404de21ff40023e6dd712b492a577416c68c6dbca1383be521e3e2e6604e4ed9aef9824031687e39b4f2c29529a6b019e388405886c27
+  metadata.gz: cfc5b645933403a7ec6b66151bcd0bb47e74324573ebe61455cbe10505e46c1ff7867960fd108b91f9a6302da8ae165ddd981f633f96175a149a74fcae6b8e70
+  data.tar.gz: 0ea95fa0ca26a6b3106f7ecd31b659025f7d7f7d76d15d0b19fd47475102135883165c5e8e655a4ceadade2ddde17ecf65fea73a1771a4a8c34f45d6f85f11b4

data/README.md

@@ -216,6 +216,102 @@ specifying the S3 bucket and object key, either of the following fields may be u
       Role: !GetAtt [ LambdaExecutionRole, Arn ]
 ```
 
+### Nested Stacks Modules
+
+The CloudFormation pre-compiler supports loading local templates as "nested stacks" using the
+CloudFormation `AWS::CloudFormation::Stack` resource type.
+
+Instead of first pre-deploying a template to S3 to be used for a nested stack, use the
+`Template` property (instead of the `TemplateURL` property) to point to a local
+sub-template. The sub-template will be compiled separately and deployed automatically to
+an S3 bucket before deploying the compiled template to CloudFormation.
+
+The `monitor` tool (also used during `create` operation) supports nested stacks by
+automatically detecting nested stack updates in the main stack's event stream and will
+start streaming the nested stack events - this allows the user to more easily locate problems
+with nested stacks.
+
+Currently there's no automatic resolution of references between nested and parent stacks, so
+make sure to set up nested stack parameters for all resources that should be referenced from
+the parent stack.
+
+#### Example
+
+`cloud-formation.yaml`:
+
+```
+AWSTemplateFormatVersion: "2010-09-09"
+Description: "CloudFormation template with nested stacks"
+Parameters:
+  DomainName:
+    Description: "The DNS domain name for the system"
+    Type: String
+    Default: example.com
+  AMI:
+    Description: "The AMI ID for the image to deploy"
+    Type: String
+    Default: ami-af4333cf
+
+Resources:
+  VPC:
+    Type: AWS::EC2::VPC
+    Properties:
+      CidrBlock: 172.20.0.0/16
+      EnableDnsSupport: true
+      EnableDnsHostnames: true
+  SecurityGroupExample:
+    Type: AWS::EC2::SecurityGroup
+    Properties:
+      VpcId: !Ref VPC 
+      GroupDescription: example security group
+      SecurityGroupIngress:
+        - { IpProtocol: icmp, CidrIp: 0.0.0.0/0, FromPort: -1, ToPort: -1 }
+        - { IpProtocol: tcp, CidrIp: 0.0.0.0/0, FromPort: 22, ToPort: 22 }
+  ServiceStack:
+    Type: AWS::CloudFormation::Stack
+    Properties:
+      Template: service.yaml
+      Parameters:
+        DomainName: !Ref DomainName
+        AMI: !Ref AMI
+        VPC: !Ref VPC
+```
+
+`service.yaml`:
+
+```
+AWSTemplateFormatVersion: "2010-09-09"
+Description: "Service nested stack"
+Parameters:
+  DomainName:
+    Description: "The DNS domain name for the system"
+    Type: String
+  AMI:
+    Description: "The AMI ID for the image to deploy"
+    Type: String
+  VPC:
+    Description: "The VPC into which to deploy the service"
+    Type: String
+
+Resources:
+  Subnet:
+    Type: AWS::EC2::Subnet
+    Properties:
+      AvailabilityZone: !Select [ 0, !GetAZs { Ref: "AWS::Region" } ]
+      CidrBlock: 172.20.0.0/24
+      MapPublicIpOnLaunch: true
+      VpcId: !Ref VPC
+  Ec2Instance: 
+    Type: AWS::EC2::Instance
+    Properties: 
+      ImageId: !Ref AMI
+      KeyName: "secret" 
+      NetworkInterfaces: 
+        - AssociatePublicIpAddress: "true"
+          DeviceIndex: "0"
+          SubnetId: !Ref Subnet 
+```
+
 ## Caching
 
 Some resource compilation may require uploading to S3, such as Lambda code or cloud-init setup
@@ -248,7 +344,12 @@ The following commands are supported:
  - `status` - Check if the names stack exists or not
  - `delete` - Delete the specified stack. After issuing the delete command, the tool will
    immediately start `monitor` mode until the operation has completed. 
- - `servers` - List EC2 instances created and managed by this stack.
+ - `servers` - List EC2 instances created and managed by this stack, per autoscaling group, including servers in nested stacks.
+ - `groups` - list autoscaling groups managed by the stack, including groups in nested stacks.
+ - `recycle` - recycle servers in an autoscaling group in a stack by scaling the group up and down.
+ - `scale` - set the scale of an autoscaling group managed by a stack to a specific desired value.
+ - `invalidate` - send an invalidation request to a CloudFront distribution managed by a stack.
+ - `output` - retrieve output values from a stack.
 
 Please see the specific help for each command by running `cftool <command> --help` for
 more details and specific options.
@@ -260,3 +361,100 @@ The AWS region to be used can be select by specifying top level option (i.e. bef
 ### Credentials Selection
 
 The tool will use the standard AWS credentials selection process, except when you want to use AWS CLI configured credential profiles, you may select to use a profile other than "default" by specifying the top level option (i.e. before the command name) `-p <profile>`, by providing the standard environment variable `AWS_DEFAULT_PROFILE` or by having a file called `.awsprofile` - whose content is the name of a valid AWS REGION - in a parent directory (at any level up to the root directory).
+
+## Library API
+
+The cloudformatin tool can also be consumed as a library by other applications - for example an application that needs to perform high-level business-logic oriented
+operations for a specific application deployed in a stack, using the cloudformation tool abstraction of CloudFormation templates and stacks.
+
+### Usage as a library
+
+To use the cloudformatin tool as a library, require `cloud_formation_tool`.
+
+### CloudFormation templates
+
+The cloudformation pre-compiler can be used to manipulate pre-compiled templates.
+
+To access the pre-compiler, initialize a `CloudFormationTool::CloudFormation` with the path to the local template resource (either a file or a directory that can be
+parsed by the pre-compiler).
+
+The initial template resource will be loaded but will not be fully parsed - and included elements will not be read - until the `compile` method is called.
+
+The following method calls are available on the `CloudFormation` instance:
+
+#### `compile(parameters = nil)`
+
+Pre-compiles the template, with the provided parameter `Hash`, if provided. Returns a `Hash` repsenting the compiled template.
+
+#### `to_yaml`
+
+Pre-compiles the template and returns a YAML rendering of the CloudFormation template, suitable for deploying to AWS CloudFormation.
+
+#### `each`
+
+Yields a tuple for each defined template parameter, that includes the parameter's name and its default value (if set, `nil` otherwise).
+
+### CloudFormation stacks
+
+The cloudformation tool's abstraction of a CloudFormation stack can be used to manipulate stack resouces, such as autoscaling groups or instances in a stack context.
+
+To access the stack API, initialize a `CloudFormationTool::CloudFormation::Stack` with the name of the stack. You can then access the following methods:
+
+#### `exist?`
+
+Check if a stack exists.
+
+#### `create(template, params = {})`
+
+Create or update a stack by deploying the specified template. The template can be any local file or directory resource that can be parsed by the cloudformation pre-compiler.
+
+#### `delete`
+
+Deletes the stack
+
+#### `stack_id`
+
+Return the AWS CloudFormation stack identifier for the stack, which is the ARN of the stack.
+
+#### `output`
+
+Returns the output values of the stack
+
+#### `resources`
+
+Return a list of resources in the stack and all of its nested stacks
+
+#### `asgroups`
+
+Return a list of autoscaling groups in the stack and all of its nested stacks. The returned values are AWS SDK CloudFormation resources, extended with a set of methods
+to help manage autoscaling groups:
+
+##### `group`
+
+Returns the AWS SDK `Aws::AutoScaling::AutoScalingGroup` object for the autoscaling group.
+
+#### `cdns`
+
+Return a list of CloudFront CDN distributions in the stack and all of its nested stacks. The returnd values are AWS SDK CloudFormation resources, extended with a set of
+methods to help manage CloudFront distributions:
+
+##### `distribution`
+
+Returns the AWS SDK `Aws::CloudFront::Types::Distribution` object for the CloudFront distribution.
+
+##### `domain_names`
+
+Returns the comma delimited list of the distribution aliases domain names
+
+##### `invalidate(path)`
+
+Creates a new invalidation in the CloudFront distribution with the specified path expression
+
+#### `each`
+
+Yields CloudFormation stack events, in the order they were created. Subsequent calls to `each` will not repeat events previously yielded and will only yield additional
+events created since the last call to `each`.
+
+#### `see_event`
+
+Mark all events since the last call to `each` (or from stack creation, if `each` was not previously called) as "seen" so they will not be yielded in future calls to `each`.

data/bin/cftool

@@ -7,9 +7,6 @@ begin
 rescue SocketError => e
   warn "Networking error: #{e.message}"
   exit 1
-rescue Seahorse::Client::NetworkingError => e
-  warn "Networking error: #{e.message}"
-  exit 1
 rescue CloudFormationTool::Errors::BaseError => e
   warn e.message
   exit 1

data/lib/cloud_formation_tool/cli/main.rb

@@ -31,7 +31,7 @@ module CloudFormationTool
       
       option [ "-d", "--debug" ], :flag, "Enable debug logging" do
         logger.level = Logger::Severity::DEBUG
-        logger.formatter = logger.default_formatter
+        logger.formatter = nil
       end
       
       option [ "-q", "--quiet" ], :flag, "Enable debug logging" do

data/lib/cloud_formation_tool/cli/servers.rb

@@ -16,13 +16,18 @@ module CloudFormationTool
           asg_name.nil? or (res.logical_resource_id == asg_name)
         end.collect do |res|
           Thread.new do
-            awsas.describe_auto_scaling_groups({
+            asg = awsas.describe_auto_scaling_groups({
               auto_scaling_group_names: [ res.physical_resource_id ]
-            }).auto_scaling_groups.first.instances.collect do |i|
-              Aws::EC2::Instance.new i.instance_id, client: awsec2
-            end.collect do |i|
-              ips = [ i.public_ip_address ] + i.network_interfaces.collect(&:ipv_6_addresses).flatten.collect(&:ipv_6_address)
-              "#{res.logical_resource_id.ljust(30, ' ')} '#{i.public_dns_name}' (#{ips.join(', ')})"
+            }).auto_scaling_groups.first
+            if asg.nil?
+              []
+            else
+              asg.instances.collect do |i|
+                Aws::EC2::Instance.new i.instance_id, client: awsec2
+              end.collect do |i|
+                ips = [ i.public_ip_address ] + i.network_interfaces.collect(&:ipv_6_addresses).flatten.collect(&:ipv_6_address)
+                "#{res.logical_resource_id.ljust(30, ' ')} '#{i.public_dns_name}' (#{ips.join(', ')})"
+              end
             end
           end
         end

data/lib/cloud_formation_tool/cloud_formation.rb

@@ -183,6 +183,12 @@ module CloudFormationTool
               else
                 load_files(val, restype)
               end
+            when 'AWS::CloudFormation::Stack'
+              if key == 'Properties' and val.key?('Template')
+                NestedStack.new(val, self).to_cloudformation
+              else
+                load_files(val, restype)
+              end
             else
               load_files(val, restype)
           end

data/lib/cloud_formation_tool/cloud_formation/nested_stack.rb

@@ -0,0 +1,25 @@
+module CloudFormationTool
+  class CloudFormation
+    
+    class NestedStack
+      include Storable
+      
+      def initialize(props, tpl)
+        @tpl = tpl
+        @data = props
+        if props.key?('Template')
+          path = props['Template']
+          path = if path.start_with? "/" then path else "#{@tpl.basedir}/#{path}" end
+          @content = CloudFormation.new(path).to_yaml
+          @data['TemplateURL'] = upload(make_filename('yaml'), @content, mime_type: 'text/yaml', gzip: false)
+          @data.delete('Template')
+        end
+      end
+      
+      def to_cloudformation
+        @data
+      end
+
+    end
+  end
+end

data/lib/cloud_formation_tool/cloud_formation/stack.rb

@@ -15,6 +15,7 @@ module CloudFormationTool
         @name = name
         @seenev = Set.new
         @watch_timeouts = 0
+        @nested_stacks = Hash[]
       end
       
       def delete
@@ -95,7 +96,14 @@ module CloudFormationTool
       def resources
         begin
           awscf.list_stack_resources(stack_name: @name).each do |resp|
-            resp.stack_resource_summaries.each { |res| yield res }
+            resp.stack_resource_summaries.each do |res|
+              yield res
+              if res.resource_type == 'AWS::CloudFormation::Stack'
+                Stack.new(res.physical_resource_id).resources do |nested_res|
+                  yield nested_res
+                end
+              end
+            end
           end
         rescue Aws::CloudFormation::Errors::ValidationError => e
           raise CloudFormationTool::Errors::AppError, "Failed to get resources: #{e.message}"
@@ -133,6 +141,7 @@ module CloudFormationTool
       end
       
       def monitor(start_time = nil)
+        @nested_stacks = Hash[]
         done = false
         begin
           until done
@@ -140,26 +149,65 @@ module CloudFormationTool
               next if @seenev.add?(ev.event_id).nil?
               text = "#{ev.timestamp.strftime "%Y-%m-%d %H:%M:%S"}| " + %w(
                 resource_type:40
-                logical_resource_id:38
+                logical_resource_id:42
                 resource_status
               ).collect { |field|
                 (name,size) = field.split(":")
                 size ||= 1
-                ev.send(name.to_sym).ljust(size.to_i, ' ')
+                (if name == 'logical_resource_id' and ev.stack_name != self.name
+                  logical_nested_stack_name(ev.stack_name) + "|"
+                else
+                  ''
+                end + ev.send(name.to_sym)).ljust(size.to_i, ' ')
               }.join("  ")
               text += " " + ev.resource_status_reason if ev.resource_status =~ /_FAILED/
               if start_time.nil? or start_time < ev.timestamp
                 puts text
               end
-              done = (ev.resource_type == "AWS::CloudFormation::Stack" and ev.resource_status =~ /(_COMPLETE|_FAILED)$/)
+              check_nested_stack(ev)
+              done = is_final_event(ev)
             end
+            sleep 1
           end
         rescue CloudFormationTool::Errors::StackDoesNotExistError => e
           puts "Stack #{name} does not exist"
         end
       end
       
+      def logical_nested_stack_name(phys_name)
+        @nested_stacks[phys_name] || 'unknown'
+      end
+      
+      def nested_stack_name(ev)
+        ev.physical_resource_id.split('/')[1]
+      end
+      
+      def check_nested_stack(ev)
+        return unless ev.resource_type == "AWS::CloudFormation::Stack" and
+          ev.logical_resource_id != self.name # not nested stack
+        return if @nested_stacks.has_key? ev.logical_resource_id # seeing the first or last nested stack event - ignoring
+        @nested_stacks[nested_stack_name(ev)] = ev.logical_resource_id
+      end
+      
+      def is_final_event(ev)
+        ev.resource_type == "AWS::CloudFormation::Stack" and
+        ev.resource_status =~ /(_COMPLETE|_FAILED)$/ and
+        ev.logical_resource_id == self.name
+      end
+      
+      def tracked_stacks
+        [ self.name ] + @nested_stacks.keys.compact
+      end
+      
       def each
+        tracked_stacks.each do |name|
+          events_for(name) do |ev|
+            yield ev
+          end
+        end
+      end
+      
+      def events_for(stack_name)
         token = nil
         sleep(if @_last_poll_time.nil?
           0
@@ -172,7 +220,7 @@ module CloudFormationTool
             end
           end)
         begin
-          resp = awscf.describe_stack_events stack_name: name, next_token: token
+          resp = awscf.describe_stack_events stack_name: stack_name, next_token: token
           @watch_timeouts = 0
           resp.stack_events.each do |ev|
             yield ev
@@ -188,7 +236,11 @@ module CloudFormationTool
           end
         rescue Aws::CloudFormation::Errors::ValidationError => e
           if e.message =~ /does not exist/
-            raise CloudFormationTool::Errors::StackDoesNotExistError, "Stack does not exist"
+            if stack_name == self.name
+              raise CloudFormationTool::Errors::StackDoesNotExistError, "Stack does not exist"
+            end
+            # ignore "does not exist" errors on nested stacks - we may try to poll them before
+            # they actually exist. We'll just try later
           else
             raise e
           end

data/lib/cloud_formation_tool/cloud_init.rb

@@ -45,7 +45,7 @@ module CloudFormationTool
     def encode(allow_gzip = true)
       yamlout = compile
       usegzip = false
-      if allow_gzip and yamlout.size > 16384 # max AWS EC2 user data size - try compressing it
+      if allow_gzip and yamlout.size > 16000 # max AWS EC2 user data size - try compressing it
         yamlout = Zlib::Deflate.new(nil, 31).deflate(yamlout, Zlib::FINISH) # 31 is the magic word to have deflate create a gzip compatible header
         usegzip = true
       end

data/lib/cloud_formation_tool/version.rb

@@ -1,3 +1,3 @@
 module CloudFormationTool
-  VERSION = '1.2.1'
+  VERSION = '1.3.4'
 end

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: cloudformation-tool
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 1.3.4
 platform: ruby
 authors:
 - Oded Arbel
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-12-05 00:00:00.000000000 Z
+date: 2020-07-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '12'
+        version: 12.3.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '12'
+        version: 12.3.3
 - !ruby/object:Gem::Dependency
   name: clamp
   requirement: !ruby/object:Gem::Requirement
@@ -168,6 +168,7 @@ files:
 - lib/cloud_formation_tool/cloud_formation.rb
 - lib/cloud_formation_tool/cloud_formation/cloud_front_distribution.rb
 - lib/cloud_formation_tool/cloud_formation/lambda_code.rb
+- lib/cloud_formation_tool/cloud_formation/nested_stack.rb
 - lib/cloud_formation_tool/cloud_formation/stack.rb
 - lib/cloud_formation_tool/cloud_init.rb
 - lib/cloud_formation_tool/errors.rb
@@ -192,8 +193,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.8
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: A pre-compiler tool for CloudFormation YAML templates