aws-cfn-dsl 0.0.4 → 0.1.0

data/lib/aws/cfn/dsl/fncall.rb

@@ -0,0 +1,21 @@
+require "aws/cfn/decompiler"
+
+module Aws
+  module Cfn
+    module Dsl
+      class FnCall
+        attr_reader :name, :arguments, :multiline
+
+        def initialize(name, arguments, multiline = false)
+          @name = name
+          @arguments = arguments
+          @multiline = multiline
+        end
+
+        def to_s()
+          @name + "(" + @arguments.join(', ') + ")"
+        end
+      end
+    end
+  end
+end

data/lib/aws/cfn/dsl/main.rb

@@ -0,0 +1,30 @@
+require 'slop'
+require "aws/cfn/dsl/base"
+
+module Aws
+  module Cfn
+    module Dsl
+      class Main < Base
+
+        def run
+
+          @opts = Slop.parse(help: true) do
+            on :j, :template=,      'The template to convert', as: String
+            on :o, :output=,        'The directory to output the DSL to.', as: String
+          end
+
+          unless @opts[:template]
+            puts @opts
+            exit
+          end
+
+          load @opts[:template]
+
+          pprint(simplify(@items))
+
+
+        end
+      end
+    end
+  end
+end

data/lib/aws/cfn/dsl/template.rb

@@ -0,0 +1,242 @@
+require 'bundler/setup'
+require 'cloudformation-ruby-dsl/cfntemplate'
+require 'cloudformation-ruby-dsl/table'
+#require 'cloudformation-ruby-dsl/spotprice'
+require 'slop'
+
+module Aws
+  module Cfn
+    module Dsl
+      class Template < ::TemplateDSL
+
+        def initialize(&block)
+          @path = File.dirname(caller[2].split(%r'\s+').shift.split(':')[0])
+          super
+        end
+
+        def file(b)
+          block = File.read File.join(@path,b)
+          eval block
+        end
+
+        def mapping(name, options=nil)
+          if options.nil?
+            file "Mappings/#{name}.rb"
+          else
+            super(name,options)
+          end
+        end
+
+        # def mapping_file(p)
+        #   file "Mappings/#{p}.rb"
+        # end
+
+        def parameter(name, options=nil)
+          if options.nil?
+            file "Parameters/#{name}.rb"
+          else
+            super(name,options)
+          end
+        end
+
+        # def parameter_file(p)
+        #   file "Parameters/#{p}.rb"
+        # end
+
+        def resource(name, options=nil)
+          if options.nil?
+            file "Resources/#{name}.rb"
+          else
+            super(name,options)
+          end
+        end
+
+        # def resource_file(p)
+        #   file "Resources/#{p}.rb"
+        # end
+
+        def output(name, options=nil)
+          if options.nil?
+            file "Outputs/#{name}.rb"
+          else
+            super(name,options)
+          end
+        end
+
+        # def output_file(p)
+        #   file "Outputs/#{p}.rb"
+        # end
+
+        def exec!(argv=ARGV)
+          @opts = Slop.parse(help: true) do
+            banner "usage: #{$PROGRAM_NAME} <expand|diff|validate|create|update|delete>"
+            on :o, :output=,        'The template file to save this DSL expansion to', as: String
+          end
+
+          action = argv[0] || 'expand'
+          unless %w(expand diff validate create update delete).include? action
+            $stderr.puts "usage: #{$PROGRAM_NAME} <expand|diff|validate|create|update|delete>"
+            exit(2)
+          end
+          unless (argv & %w(--template-file --template-url)).empty?
+            $stderr.puts "#{File.basename($PROGRAM_NAME)}:  The --template-file and --template-url command-line options are not allowed. (You are running the template itself right now ... !)"
+            exit(2)
+          end
+
+          # Find parameters where extension attribute :Immutable is true then remove it from the
+          # cfn template since we can't pass it to CloudFormation.
+          immutable_parameters = excise_parameter_attribute!(:Immutable)
+
+          # Tag CloudFormation stacks based on :Tags defined in the template
+          cfn_tags = excise_tags!
+          # The command line string looks like: --tag "Key=key; Value=value" --tag "Key2=key2; Value2=value"
+          cfn_tags_options = cfn_tags.sort.map { |tag| ["--tag", "Key=%s; Value=%s" % tag.split('=')] }.flatten
+
+          # example: <template.rb> cfn-create-stack my-stack-name --parameters "Env=prod" --region eu-west-1
+          # Execute the AWS CLI cfn-cmd command to validate/create/update a CloudFormation stack.
+          if action == 'diff' or (action == 'expand' and not nopretty)
+            template_string = JSON.pretty_generate(self)
+          else
+            template_string = JSON.generate(self)
+          end
+
+          if action == 'expand'
+            # Write the pretty-printed JSON template to stdout and exit.  [--nopretty] option writes output with minimal whitespace
+            # example: <template.rb> expand --parameters "Env=prod" --region eu-west-1 --nopretty
+            if @opts[:output]
+              dest = @opts[:output]
+              if File.directory? dest
+                file = File.basename $PROGRAM_NAME
+                file.gsub!(%r'\.rb', '.json')
+                dest = File.join dest, file
+              end
+              IO.write(dest, template_string)
+            else
+              puts template_string
+            end
+            exit(true)
+          end
+
+          temp_file = File.absolute_path("#{$PROGRAM_NAME}.expanded.json")
+          File.write(temp_file, template_string)
+
+          cmdline = ['cfn-cmd'] + argv + ['--template-file', temp_file] + cfn_tags_options
+
+          case action
+            when 'diff'
+              # example: <template.rb> diff my-stack-name --parameters "Env=prod" --region eu-west-1
+              # Diff the current template for an existing stack with the expansion of this template.
+
+              # The --parameters and --tag options were used to expand the template but we don't need them anymore.  Discard.
+              _, cfn_options = extract_options(argv[1..-1], %w(), %w(--parameters --tag))
+
+              # Separate the remaining command-line options into options for 'cfn-cmd' and options for 'diff'.
+              cfn_options, diff_options = extract_options(cfn_options, %w(),
+                                                          %w(--stack-name --region --parameters --connection-timeout -I --access-key-id -S --secret-key -K --ec2-private-key-file-path -U --url))
+
+              # If the first argument is a stack name then shift it from diff_options over to cfn_options.
+              if diff_options[0] && !(/^-/ =~ diff_options[0])
+                cfn_options.unshift(diff_options.shift)
+              end
+
+              # Run CloudFormation commands to describe the existing stack
+              cfn_options_string           = cfn_options.map { |arg| "'#{arg}'" }.join(' ')
+              old_template_raw             = exec_capture_stdout("cfn-cmd cfn-get-template #{cfn_options_string}")
+              # ec2 template output is not valid json: TEMPLATE  "<json>\n"\n
+              old_template_object          = JSON.parse(old_template_raw[11..-3])
+              old_template_string          = JSON.pretty_generate(old_template_object)
+              old_stack_attributes         = exec_describe_stack(cfn_options_string)
+              old_tags_string              = old_stack_attributes["TAGS"]
+              old_parameters_string        = old_stack_attributes["PARAMETERS"]
+
+              # Sort the tag strings alphabetically to make them easily comparable
+              old_tags_string = (old_tags_string || '').split(';').sort.map { |tag| %Q(TAG "#{tag}"\n) }.join
+              tags_string     = cfn_tags.sort.map { |tag| "TAG \"#{tag}\"\n" }.join
+
+              # Sort the parameter strings alphabetically to make them easily comparable
+              old_parameters_string = (old_parameters_string || '').split(';').sort.map { |param| %Q(PARAMETER "#{param}"\n) }.join
+              parameters_string     = parameters.sort.map { |key, value| "PARAMETER \"#{key}=#{value}\"\n" }.join
+
+              # Diff the expanded template with the template from CloudFormation.
+              old_temp_file = File.absolute_path("#{$PROGRAM_NAME}.current.json")
+              new_temp_file = File.absolute_path("#{$PROGRAM_NAME}.expanded.json")
+              File.write(old_temp_file, old_tags_string + old_parameters_string + old_template_string)
+              File.write(new_temp_file, tags_string + parameters_string + template_string)
+
+              # Compare templates
+              system(*["diff"] + diff_options + [old_temp_file, new_temp_file])
+
+              File.delete(old_temp_file)
+              File.delete(new_temp_file)
+
+              exit(true)
+
+            when 'cfn-validate-template'
+              # The cfn-validate-template command doesn't support --parameters so remove it if it was provided for template expansion.
+              _, cmdline = extract_options(cmdline, %w(), %w(--parameters --tag))
+
+            when 'cfn-update-stack'
+              # Pick out the subset of cfn-update-stack options that apply to cfn-describe-stacks.
+              cfn_options, other_options = extract_options(argv[1..-1], %w(),
+                                                           %w(--stack-name --region --connection-timeout -I --access-key-id -S --secret-key -K --ec2-private-key-file-path -U --url))
+
+              # If the first argument is a stack name then shift it over to cfn_options.
+              if other_options[0] && !(/^-/ =~ other_options[0])
+                cfn_options.unshift(other_options.shift)
+              end
+
+              # Run CloudFormation command to describe the existing stack
+              cfn_options_string = cfn_options.map { |arg| "'#{arg}'" }.join(' ')
+              old_stack_attributes = exec_describe_stack(cfn_options_string)
+
+              # If updating a stack and some parameters are marked as immutable, fail if the new parameters don't match the old ones.
+              if not immutable_parameters.empty?
+                old_parameters_string = old_stack_attributes["PARAMETERS"]
+                old_parameters = Hash[(old_parameters_string || '').split(';').map { |pair| pair.split('=', 2) }]
+                new_parameters = parameters
+
+                immutable_parameters.sort.each do |param|
+                  if old_parameters[param].to_s != new_parameters[param].to_s
+                    $stderr.puts "Error: cfn-update-stack may not update immutable parameter " +
+                                     "'#{param}=#{old_parameters[param]}' to '#{param}=#{new_parameters[param]}'."
+                    exit(false)
+                  end
+                end
+              end
+
+              # Tags are immutable in CloudFormation.  The cfn-update-stack command doesn't support --tag options, so remove
+              # the argument (if it exists) and validate against the existing stack to ensure tags haven't changed.
+              # Compare the sorted arrays for an exact match
+              old_cfn_tags = old_stack_attributes['TAGS'].split(';').sort rescue [] # Use empty Array if .split fails
+              if cfn_tags != old_cfn_tags
+                $stderr.puts "CloudFormation stack tags do not match and cannot be updated. You must either use the same tags or create a new stack." +
+                                 "\n" + (old_cfn_tags - cfn_tags).map {|tag| "< #{tag}" }.join("\n") +
+                                 "\n" + "---" +
+                                 "\n" + (cfn_tags - old_cfn_tags).map {|tag| "> #{tag}"}.join("\n")
+                exit(false)
+              end
+              _, cmdline = extract_options(cmdline, %w(), %w(--tag))
+          end
+
+          # Execute command cmdline
+          unless system(*cmdline)
+            $stderr.puts "\nExecution of 'cfn-cmd' failed.  To facilitate debugging, the generated JSON template " +
+                             "file was not deleted.  You may delete the file manually if it isn't needed: #{temp_file}"
+            exit(false)
+          end
+
+          File.delete(temp_file)
+
+          exit(true)
+        end
+
+
+      end
+    end
+  end
+end
+
+# Main entry point
+def template(&block)
+  Aws::Cfn::Dsl::Template.new(&block)
+end

data/lib/aws/cfn/dsl/version.rb

@@ -1,7 +1,7 @@
 module Aws
   module Cfn
     module Dsl
-      VERSION = "0.0.4"
+      VERSION = "0.1.0"
     end
   end
 end

data/test/Mappings/AWSRegion2AMI.rb

@@ -0,0 +1,10 @@
+  mapping 'AWSRegion2AMI',
+          :'us-east-1' => { :id => 'ami-83dee0ea' },
+          :'us-west-1' => { :id => 'ami-c45f6281' },
+          :'us-west-2' => { :id => 'ami-d0d8b8e0' },
+          :'eu-west-1' => { :id => 'ami-aa56a1dd' },
+          :'sa-east-1' => { :id => 'ami-d55bfbc8' },
+          :'ap-southeast-1' => { :id => 'ami-bc7325ee' },
+          :'ap-southeast-2' => { :id => 'ami-e577e9df' },
+          :'ap-northeast-1' => { :id => 'ami-f72e45f6' }
+

data/test/Outputs/ChefSecurityGroup.rb

@@ -0,0 +1,4 @@
+  output 'ChefSecurityGroup',
+         :Description => 'EC2 Security Group with access to Opscode chef server',
+         :Value => { :Ref => 'ChefClientSecurityGroup' }
+

data/test/Outputs/ServerURL.rb

@@ -0,0 +1,13 @@
+  output 'ServerURL',
+         :Description => 'URL of newly created Opscode chef server',
+         :Value => {
+             :'Fn::Join' => [
+                 '',
+                 [
+                     'https://',
+                     { :'Fn::GetAtt' => [ 'ChefServer', 'PublicDnsName' ] },
+                     ':443',
+                 ],
+             ],
+         }
+

data/test/Outputs/ValidationKeyBucket.rb

@@ -0,0 +1,4 @@
+  output 'ValidationKeyBucket',
+         :Description => 'Location of validation key',
+         :Value => { :Ref => 'PrivateKeyBucket' }
+

data/test/Parameters/CookbookLocation.rb

@@ -0,0 +1,5 @@
+  parameter 'CookbookLocation',
+            :Type => 'String',
+            :Default => 'https://github.com/opscode-cookbooks/aws/tarball/master',
+            :Description => 'Location of chef cookbooks to upload to server'
+

data/test/Parameters/InstanceType.rb

@@ -0,0 +1,7 @@
+  parameter 'InstanceType',
+            :Description => 'WebServer EC2 instance type',
+            :Type => 'String',
+            :Default => 'm1.small',
+            :AllowedValues => %w(t1.micro m1.small m1.medium m1.large m1.xlarge m2.xlarge m2.2xlarge m2.4xlarge m3.xlarge m3.2xlarge c1.medium c1.xlarge cc1.4xlarge cc2.8xlarge cg1.4xlarge),
+            :ConstraintDescription => 'must be a valid EC2 instance type.'
+

data/test/Parameters/KeyName.rb

@@ -0,0 +1,8 @@
+  parameter 'KeyName',
+            :Description => 'Name of an existing EC2 KeyPair to enable SSH access to the web server',
+            :Type => 'String',
+            :MinLength => '1',
+            :MaxLength => '255',
+            :AllowedPattern => '[\\x20-\\x7E]*',
+            :ConstraintDescription => 'can contain only ASCII characters.'
+

data/test/Parameters/RoleLocation.rb

@@ -0,0 +1,5 @@
+  parameter 'RoleLocation',
+            :Type => 'String',
+            :Default => 'https://s3.amazonaws.com/cloudformation-examples/example_chef_roles.tar.gz',
+            :Description => 'Location of client roles to upload to server'
+

data/test/Parameters/SSHLocation.rb

@@ -0,0 +1,9 @@
+  parameter 'SSHLocation',
+            :Description => 'The IP address range that can be used to SSH to the EC2 instances',
+            :Type => 'String',
+            :MinLength => '9',
+            :MaxLength => '18',
+            :Default => '0.0.0.0/0',
+            :AllowedPattern => '(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})',
+            :ConstraintDescription => 'must be a valid IP CIDR range of the form x.x.x.x/x.'
+

data/test/Resources/BucketPolicy.rb

@@ -0,0 +1,28 @@
+  resource 'BucketPolicy', :Type => 'AWS::S3::BucketPolicy', :Properties => {
+      :PolicyDocument => {
+          :Version => '2008-10-17',
+          :Id => 'WritePolicy',
+          :Statement => [
+              {
+                  :Sid => 'WriteAccess',
+                  :Action => [ 's3:PutObject' ],
+                  :Effect => 'Allow',
+                  :Resource => {
+                      :'Fn::Join' => [
+                          '',
+                          [
+                              'arn:aws:s3:::',
+                              { :Ref => 'PrivateKeyBucket' },
+                              '/*',
+                          ],
+                      ],
+                  },
+                  :Principal => {
+                      :AWS => { :'Fn::GetAtt' => [ 'ChefServerUser', 'Arn' ] },
+                  },
+              },
+          ],
+      },
+      :Bucket => { :Ref => 'PrivateKeyBucket' },
+  }
+

data/test/Resources/ChefClientSecurityGroup.rb

@@ -0,0 +1,2 @@
+  resource 'ChefClientSecurityGroup', :Type => 'AWS::EC2::SecurityGroup', :Properties => { :GroupDescription => 'Group with access to Chef Server' }
+

data/test/Resources/ChefServer.rb

@@ -0,0 +1,57 @@
+  resource 'ChefServer', :Type => 'AWS::EC2::Instance', :Metadata => { :'AWS::CloudFormation::Init' => { :config => { :packages => { :apt => { :s3cmd => [] } }, :sources => { :'/home/ubuntu/chef-repo' => 'https://github.com/opscode/chef-repo/tarball/master', :'/home/ubuntu/chef-repo/cookbooks' => { :Ref => 'CookbookLocation' }, :'/home/ubuntu/chef-repo/roles' => { :Ref => 'RoleLocation' } }, :files => { :'/home/ubuntu/setup_environment' => { :source => 'https://s3.amazonaws.com/cloudformation-examples/setup-chef-server-with-knife', :mode => '000755', :owner => 'ubuntu', :group => 'ubuntu' }, :'/home/ubuntu/.s3cfg' => { :content => { :'Fn::Join' => [ '', [ "[default]\n", 'access_key = ', { :Ref => 'HostKeys' }, "\n", 'secret_key = ', { :'Fn::GetAtt' => [ 'HostKeys', 'SecretAccessKey' ] }, "\n", "use_https = True\n" ] ] }, :mode => '000644', :owner => 'ubuntu', :group => 'ubuntu' }, :'/home/ubuntu/chef_11.10.0-1.ubuntu.12.04_amd64.deb' => { :source => 'https://s3.amazonaws.com/cloudformation-examples/chef_11.10.0-1.ubuntu.12.04_amd64.deb', :mode => '000664', :owner => 'ubuntu', :group => 'ubuntu' }, :'/home/ubuntu/chef-server_11.0.10-1.ubuntu.12.04_amd64.deb' => { :source => 'https://s3.amazonaws.com/cloudformation-examples/chef-server_11.0.10-1.ubuntu.12.04_amd64.deb', :mode => '000664', :owner => 'ubuntu', :group => 'ubuntu' } } } } }, :Properties => {
+      :SecurityGroups => [
+          { :Ref => 'ChefServerSecurityGroup' },
+      ],
+      :ImageId => {
+          :'Fn::FindInMap' => [
+              'AWSRegion2AMI',
+              { :Ref => 'AWS::Region' },
+              'id',
+          ],
+      },
+      :UserData => {
+          :'Fn::Base64' => {
+              :'Fn::Join' => [
+                  '',
+                  [
+                      "#!/bin/bash\n",
+                      "function error_exit\n",
+                      "{\n",
+                      '  cfn-signal -e 1 -r "$1" \'',
+                      { :Ref => 'ChefServerWaitHandle' },
+                      "'\n",
+                      "  exit 1\n",
+                      "}\n",
+                      "apt-get -y install python-setuptools\n",
+                      "easy_install https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz\n",
+                      'cfn-init --region ',
+                      { :Ref => 'AWS::Region' },
+                      '    -s ',
+                      { :Ref => 'AWS::StackId' },
+                      ' -r ChefServer ',
+                      "|| error_exit 'Failed to run cfn-init'\n",
+                      "# Bootstrap chef\n",
+                      "dpkg -i /home/ubuntu/chef_11.10.0-1.ubuntu.12.04_amd64.deb > /tmp/chef_install.log 2>&1 || error_exit 'Failed to install chef client tools'\n",
+                      "dpkg -i /home/ubuntu/chef-server_11.0.10-1.ubuntu.12.04_amd64.deb >> /tmp/chef_install.log 2>&1 || error_exit 'Failed to install chef server'\n",
+                      "sleep 5\n",
+                      "sudo /usr/bin/chef-server-ctl reconfigure > /tmp/chef_configure.log 2>&1 || error_exit 'Failed to configure chef server'\n",
+                      "sleep 5\n",
+                      "sudo chef-server-ctl start\n",
+                      "# Setup development environment in ubuntu user\n",
+                      "sudo -u ubuntu /home/ubuntu/setup_environment > /tmp/setup_environment.log 2>&1 || error_exit 'Failed to bootstrap chef server'\n",
+                      "# copy validation key to S3 bucket\n",
+                      's3cmd -c /home/ubuntu/.s3cfg put /etc/chef-server/validation.pem s3://',
+                      { :Ref => 'PrivateKeyBucket' },
+                      "/validation.pem > /tmp/put_validation_key.log 2>&1 || error_exit 'Failed to put Chef Server validation key'\n",
+                      "# If all went well, signal success\n",
+                      'cfn-signal -e $? -r \'Chef Server configuration\' \'',
+                      { :Ref => 'ChefServerWaitHandle' },
+                      "'\n",
+                  ],
+              ],
+          },
+      },
+      :KeyName => { :Ref => 'KeyName' },
+      :InstanceType => { :Ref => 'InstanceType' },
+  }
+