enscalator 0.4.0.pre.alpha.pre.16

data/lib/enscalator/enapp.rb

@@ -0,0 +1,248 @@
+require 'cloudformation-ruby-dsl/cfntemplate'
+require_relative 'rich_template_dsl'
+
+module Enscalator
+  # Template DSL for common enJapan application stack
+  class EnAppTemplateDSL < RichTemplateDSL
+    include Enscalator::Helpers
+
+    attr_reader :app_name
+
+    Struct.new('Subnet', :availability_zone, :suffix, :cidr_block)
+
+    # Subnet size (256 addresses)
+    SUBNET_CIDR_BLOCK_SIZE = 24
+
+    # Create new EnAppTemplateDSL instance
+    #
+    # @param [Hash] options command-line arguments
+    def initialize(options = {})
+      # application name taken from template name by default
+      @app_name = self.class.name.demodulize
+      super
+    end
+
+    # Get vpc stack
+    #
+    # @return [Aws::CloudFormation::Stack] stack instance of vpc stack
+    def vpc_stack
+      @vpc_stack ||= cfn_resource(cfn_client(region)).stack(vpc_stack_name)
+    end
+
+    # Get current stack
+    #
+    # @return [Aws::CloudFormation::Stack] current stack
+    def current_stack
+      @current_stack ||= (cfn_resource(cfn_client(region)).stack(stack_name) rescue nil) unless creating?
+    end
+
+    # Get vpc
+    #
+    # @return [Aws::EC2::Vpc] vpc instance
+    def vpc
+      @vpc ||= Aws::EC2::Vpc.new(id: get_resource(vpc_stack, 'VPC'), region: region)
+    end
+
+    # References to application subnets in all availability zones
+    def ref_application_subnets
+      availability_zones.map { |suffix, _| ref("ApplicationSubnet#{suffix.upcase}") }
+    end
+
+    # References to resource subnets in all availability zones
+    def ref_resource_subnets
+      availability_zones.map { |suffix, _| ref("ResourceSubnet#{suffix.upcase}") }
+    end
+
+    # Public subnets in all availability zones
+    def public_subnets
+      availability_zones.map { |suffix, _| get_resource(vpc_stack, "PublicSubnet#{suffix.upcase}") }
+    end
+
+    # Get VPC ID as reference to parameter
+    # @return [Hash]
+    def ref_vpc_id
+      ref('VpcId')
+    end
+
+    # Reference to private security group
+    # @return [Hash]
+    def ref_private_security_group
+      ref('PrivateSecurityGroup')
+    end
+
+    # Reference to resource security group
+    # @return [Hash]
+    def ref_resource_security_group
+      ref('ResourceSecurityGroup')
+    end
+
+    # Reference to application security group
+    # @return [Hash]
+    def ref_application_security_group
+      ref('ApplicationSecurityGroup')
+    end
+
+    # Get all CIRD blocks for current VPC
+    # @return [Hash]
+    def get_all_cidr_blocks
+      IPAddress(
+        Core::NetworkConfig.mapping_vpc_net[region.to_sym][:VPC]).subnet(SUBNET_CIDR_BLOCK_SIZE).map(&:to_string)
+    end
+
+    # Get currently used CIDR blocks
+    # @return [Array]
+    def get_used_cidr_blocks
+      vpc.subnets.collect(&:cidr_block)
+    end
+
+    # Get non-used CIDR blocks
+    # @return [Array]
+    def get_available_cidr_blocks
+      get_all_cidr_blocks - get_used_cidr_blocks
+    end
+
+    # Get application CIDR blocks availability zones mapping
+    # @return [Hash]
+    def get_application_to_az_mapping
+      cidr_blocks = get_available_cidr_blocks.dup
+      availability_zones.map do |suffix, az|
+        cidr_block = (begin
+          subnet_id = get_resource(current_stack, "ApplicationSubnet#{suffix.upcase}")
+          Aws::EC2::Subnet.new(id: subnet_id, region: region).cidr_block
+        end rescue nil) if current_stack
+
+        Struct::Subnet.new(az, suffix, cidr_block || cidr_blocks.shift)
+      end
+    end
+
+    # CIDR blocks allocated for application subnets
+    # @return [Array]
+    def get_application_cidr_blocks
+      get_application_to_az_mapping.map(&:cidr_block)
+    end
+
+    # Get resource CIDR blocks availability zones mapping
+    # @return [Array]
+    def get_resource_to_az_mapping
+      cidr_blocks = (get_available_cidr_blocks - get_application_cidr_blocks).dup
+      availability_zones.map do |suffix, az|
+        cidr_block = (begin
+          subnet_id = get_resource(current_stack, "ResourceSubnet#{suffix.upcase}")
+          Aws::EC2::Subnet.new(id: subnet_id, region: region).cidr_block
+        end rescue nil) if current_stack
+
+        Struct::Subnet.new(az, suffix, cidr_block || cidr_blocks.shift)
+      end
+    end
+
+    # CIDR blocks allocated for resource subnets
+    # @return [Array]
+    def get_resource_cidr_blocks
+      get_resource_to_az_mapping.map(&:cidr_block)
+    end
+
+    # Query and pre-configure VPC parameters required for the stack
+    def load_vpc_params
+      parameter 'VpcId',
+                Description: 'The Id of the VPC',
+                Default: vpc.id,
+                Type: 'String',
+                AllowedPattern: 'vpc-[a-zA-Z0-9]*',
+                ConstraintDescription: 'must begin with vpc- followed by numbers and alphanumeric characters.'
+
+      parameter 'PrivateSecurityGroup',
+                Description: 'Security group identifier of private instances',
+                Default: get_resource(vpc_stack, 'PrivateSecurityGroup'),
+                Type: 'String',
+                AllowedPattern: 'sg-[a-zA-Z0-9]*',
+                ConstraintDescription: 'must begin with sg- followed by numbers and alphanumeric characters.'
+
+      # allocate application/resource cidr blocks dynamically for all availability zones
+      availability_zones.zip(get_application_cidr_blocks,
+                             get_resource_cidr_blocks).each do |pair, application_cidr_block, resource_cidr_block|
+        suffix, availability_zone = pair
+
+        private_route_table_name = "PrivateRouteTable#{suffix.upcase}"
+        parameter private_route_table_name,
+                  Description: "Route table identifier for private instances of zone #{suffix}",
+                  Default: get_resource(vpc_stack, private_route_table_name),
+                  Type: 'String',
+                  AllowedPattern: 'rtb-[a-zA-Z0-9]*',
+                  ConstraintDescription: 'must begin with rtb- followed by numbers and alphanumeric characters.'
+
+        application_subnet_name = "ApplicationSubnet#{suffix.upcase}"
+        subnet application_subnet_name,
+               vpc.id,
+               application_cidr_block,
+               availability_zone: availability_zone,
+               tags: {
+                 Network: 'Private',
+                 Application: aws_stack_name,
+                 immutable_metadata: join('', '{ "purpose": "', aws_stack_name, '-app" }')
+               }
+
+        resource_subnet_name = "ResourceSubnet#{suffix.upcase}"
+        subnet resource_subnet_name,
+               vpc.id,
+               resource_cidr_block,
+               availability_zone: availability_zone,
+               tags: {
+                 Network: 'Private',
+                 Application: aws_stack_name
+               }
+
+        resource "ApplicationRouteTableAssociation#{suffix.upcase}",
+                 Type: 'AWS::EC2::SubnetRouteTableAssociation',
+                 Properties: {
+                   RouteTableId: ref(private_route_table_name),
+                   SubnetId: ref(application_subnet_name)
+                 }
+
+        resource "ResourceRouteTableAssociation#{suffix.upcase}",
+                 Type: 'AWS::EC2::SubnetRouteTableAssociation',
+                 Properties: {
+                   RouteTableId: ref(private_route_table_name),
+                   SubnetId: ref(resource_subnet_name)
+                 }
+      end
+
+      security_group_vpc 'ResourceSecurityGroup',
+                         'Enable internal access with ssh',
+                         vpc.id,
+                         security_group_ingress: [
+                           {
+                             IpProtocol: 'tcp',
+                             FromPort: '22',
+                             ToPort: '22',
+                             CidrIp: '10.0.0.0/8'
+                           },
+                           {
+                             IpProtocol: 'tcp',
+                             FromPort: '0',
+                             ToPort: '65535',
+                             SourceSecurityGroupId: ref_application_security_group
+                           }
+                         ],
+                         tags: {
+                           Name: join('-', aws_stack_name, 'res', 'sg'),
+                           Application: aws_stack_name
+                         }
+
+      security_group_vpc 'ApplicationSecurityGroup',
+                         'Security group of the application servers',
+                         vpc.id,
+                         security_group_ingress: [
+                           {
+                             IpProtocol: 'tcp',
+                             FromPort: '0',
+                             ToPort: '65535',
+                             CidrIp: '10.0.0.0/8'
+                           }
+                         ],
+                         tags: {
+                           Name: join('-', aws_stack_name, 'app', 'sg'),
+                           Application: aws_stack_name
+                         }
+    end
+  end # class EnAppTemplateDSL
+end # module Enscalator

data/lib/enscalator/helpers/dns.rb

@@ -0,0 +1,62 @@
+module Enscalator
+  module Helpers
+    module Dns
+      # Get existing DNS records
+      #
+      # @param [String] zone_name name of the hosted zone
+      def get_dns_records(zone_name: nil)
+        client = route53_client(nil)
+        zone = client.list_hosted_zones[:hosted_zones].find { |x| x.name == zone_name }
+        records = client.list_resource_record_sets(hosted_zone_id: zone.id)
+        records.values.flatten.map do |x|
+          {
+            name: x.name,
+            type: x.type,
+            records: x.resource_records.map(&:value)
+          } if x.is_a?(Aws::Structure)
+        end.compact
+      end
+
+      # Create DNS record in given hosted zone
+      #
+      # @param [String] region aws valid region identifier
+      # @param [String] zone_name name of the hosted zone
+      # @param [String] record_name name of the dns record
+      # @param [String] type record type (NS, MX, CNAME and etc.)
+      # @param [Array] values list of record values
+      # @param [Integer] ttl time to live
+      # @param [String] suffix additional identifier following region
+      def upsert_dns_record(region: nil,
+                            zone_name: nil,
+                            record_name: nil,
+                            type: 'A',
+                            values: [],
+                            ttl: 300,
+                            suffix: '')
+        client = route53_client(region: region)
+        zone = client.list_hosted_zones[:hosted_zones].find { |x| x.name == zone_name }
+        record_tokens = [record_name.gsub(zone_name, ''), region]
+        record_tokens << suffix if suffix && !suffix.empty?
+        record_name = [record_tokens.join, zone_name].join('.')
+
+        client.change_resource_record_sets(
+          hosted_zone_id: zone.id,
+          change_batch: {
+            comment: "dns record for #{record_name}",
+            changes: [
+              {
+                action: 'UPSERT',
+                resource_record_set: {
+                  name: record_name,
+                  type: type,
+                  resource_records: values.map { |x| { value: x } },
+                  ttl: ttl
+                }
+              }
+            ]
+          }
+        )
+      end
+    end
+  end
+end

data/lib/enscalator/helpers/stack.rb

@@ -0,0 +1,107 @@
+require 'ruby-progressbar'
+
+module Enscalator
+  module Helpers
+    # Helpers for operations requiring stack instance or stack_name
+    module Stack
+      # Wait until stack gets created
+      #
+      # @param [Aws::CloudFormation::Resource] cfn accessor for cloudformation resource
+      # @param [String] stack_name name of the stack
+      # @return [Aws::CloudFormation::Stack]
+      def wait_stack(cfn, stack_name)
+        stack = cfn.stack(stack_name)
+        title = 'Waiting for stack to be created'
+        progress = ProgressBar.create(title: title, starting_at: 10, total: nil)
+        loop do
+          break unless stack.stack_status =~ /(CREATE|UPDATE)_IN_PROGRESS$/
+          progress.title = title + " [#{stack.stack_status}]"
+          progress.increment
+          sleep 5
+          stack = cfn.stack(stack_name)
+        end
+        stack
+      end
+
+      # Create stack using cloudformation interface
+      #
+      # @param [String] region AWS region identifier
+      # @param [String] dependent_stack_name name of the stack current stack depends on
+      # @param [String] template name
+      # @param [String] stack_name stack name
+      # @param [Array] keys keys
+      # @param [Array] extra_parameters additional parameters
+      # @return [Aws::CloudFormation::Resource]
+      # @deprecated this method is no longer used
+      def cfn_create_stack(region, dependent_stack_name, template, stack_name, keys: [], extra_parameters: [])
+        cfn = cfn_resource(cfn_client(region))
+        stack = wait_stack(cfn, dependent_stack_name)
+        extra_parameters_cleaned = extra_parameters.map do |x|
+          if x.key? 'ParameterKey'
+            {
+              parameter_key: x['ParameterKey'],
+              parameter_value: x['ParameterValue']
+            }
+          else
+            x
+          end
+        end
+        options = {
+          stack_name: stack_name,
+          template_body: template,
+          parameters: generate_parameters(stack, keys) + extra_parameters_cleaned
+        }
+        cfn.create_stack(options)
+      end
+
+      # Get resource for given key from given stack
+      #
+      # @param [Aws::CloudFormation::Stack] stack cloudformation stack instance
+      # @param [String] key resource identifier (key)
+      # @return [String] AWS resource identifier
+      # @raise [ArgumentError] when stack is nil
+      # @raise [ArgumentError] when key is nil or empty
+      def get_resource(stack, key)
+        fail ArgumentError, 'stack must not be nil' if stack.nil?
+        fail ArgumentError, 'key must not be nil nor empty' if key.nil? || key.empty?
+        # query with physical_resource_id
+        resource = begin
+          stack.resource(key).physical_resource_id
+        rescue RuntimeError
+          nil
+        end
+        if resource.nil?
+          # fallback to values from stack.outputs
+          output = stack.outputs.select { |s| s.output_key == key }
+          resource = begin
+            output.first.output_value
+          rescue RuntimeError
+            nil
+          end
+        end
+        resource
+      end
+
+      # Get list of resources for given keys
+      #
+      # @param [Aws::CloudFormation::Stack] stack cloudformation stack instance
+      # @param [Array] keys list of resource identifiers (keys)
+      # @return [String] list of AWS resource identifiers
+      # @raise [ArgumentError] when stack is nil
+      # @raise [ArgumentError] when keys are nil or empty list
+      def get_resources(stack, keys)
+        fail ArgumentError, 'stack must not be nil' if stack.nil?
+        fail ArgumentError, 'key must not be nil nor empty' if keys.nil? || keys.empty?
+        keys.map { |k| get_resource(stack, k) }.compact
+      end
+
+      # Generate parameters list
+      #
+      # @param [Aws::CloudFormation::Stack] stack cloudformation stack instance
+      # @param [Array] keys list of keys
+      def generate_parameters(stack, keys)
+        keys.map { |k| { parameter_key: k, parameter_value: get_resource(stack, k) } }
+      end
+    end
+  end
+end

data/lib/enscalator/helpers/sub_process.rb

@@ -0,0 +1,72 @@
+require 'open3'
+
+module Enscalator
+  module Helpers
+    # Executed command as sub-processes with stdout and stderr streams
+    #  taken from: https://nickcharlton.net/posts/ruby-subprocesses-with-stdout-stderr-streams.html
+    class SubProcess
+      # Create new subprocess and execute command there
+      #
+      # @param [String] cmd command to be executed
+      def initialize(cmd)
+        # standard input is not used
+        Open3.popen3(cmd) do |_stdin, stdout, stderr, thread|
+          { out: stdout, err: stderr }.each do |key, stream|
+            Thread.new do
+              until (line = stream.gets).nil?
+                # yield the block depending on the stream
+                if key == :out
+                  yield line, nil, thread if block_given?
+                else
+                  yield nil, line, thread if block_given?
+                end
+              end
+            end
+          end
+          thread.join # wait for external process to finish
+        end
+      end
+    end
+
+    # Call script
+    #
+    # @param [String] region AWS region identifier
+    # @param [String] dependent_stack_name name of the stack current stack depends on
+    # @param [String] script_path path to script
+    # @param [Array] keys list of keys
+    # @param [String] prepend_args prepend arguments
+    # @param [String] append_args append arguments
+    # @deprecated this method is no longer used
+    def cfn_call_script(region,
+                        dependent_stack_name,
+                        script_path,
+                        keys,
+                        prepend_args: '',
+                        append_args: '')
+      cfn = cfn_resource(cfn_client(region))
+      stack = wait_stack(cfn, dependent_stack_name)
+      args = get_resources(stack, keys).join(' ')
+      cmd = [script_path, prepend_args, args, append_args]
+      begin
+        run_cmd(cmd)
+      rescue Errno::ENOENT
+        puts $ERROR_INFO.to_s
+        STDERR.puts cmd
+      end
+    end
+
+    # Run command and print captured output to corresponding standard streams
+    #
+    # @param [Array] cmd command array to be executed
+    # @return [String] produced output from executed command
+    def run_cmd(cmd)
+      # use contracts to get rid of exceptions: https://github.com/egonSchiele/contracts.ruby
+      fail ArgumentError, "Expected Array, but actually was given #{cmd.class}" unless cmd.is_a?(Array)
+      fail ArgumentError, 'Argument cannot be empty' if cmd.empty?
+      SubProcess.new(cmd.join(' ')) do |stdout, stderr, _thread|
+        STDOUT.puts stdout if stdout
+        STDERR.puts stderr if stderr
+      end
+    end
+  end
+end

data/lib/enscalator/helpers/wrappers.rb

@@ -0,0 +1,55 @@
+module Enscalator
+  module Helpers
+    # Helpers that wrap some
+    module Wrappers
+      # Cloudformation client
+      #
+      # @param [String] region Region in Amazon AWS
+      # @return [Aws::CloudFormation::Client]
+      # @raise [ArgumentError] when region is not given
+      def cfn_client(region)
+        fail ArgumentError, 'Unable to proceed without region' if region.blank? && !Aws.config.key?(:region)
+        opts = {}
+        opts[:region] = region unless Aws.config.key?(:region)
+        Aws::CloudFormation::Client.new(opts)
+      end
+
+      # EC2 client
+      #
+      # @param [String] region Region in Amazon AWS
+      # @return [Aws::EC2::Client]
+      # @raise [ArgumentError] when region is not given
+      def ec2_client(region)
+        fail ArgumentError, 'Unable to proceed without region' if region.blank? && !Aws.config.key?(:region)
+        opts = {}
+        opts[:region] = region unless Aws.config.key?(:region)
+        # noinspection RubyArgCount
+        Aws::EC2::Client.new(opts)
+      end
+
+      # Route 53 client
+      #
+      # @param [String] region AWS region identifier
+      # @return [Aws::Route53::Client]
+      # @raise [ArgumentError] when region is not given
+      def route53_client(region)
+        fail ArgumentError, 'Unable to proceed without region' if region.blank? && !Aws.config.key?(:region)
+        opts = {}
+        opts[:region] = region unless Aws.config.key?(:region)
+        # noinspection RubyArgCount
+        Aws::Route53::Client.new(opts)
+      end
+
+      # Cloudformation resource
+      #
+      # @param [Aws::CloudFormation::Client] client instance of AWS Cloudformation client
+      # @return [Aws::CloudFormation::Resource]
+      # @raise [ArgumentError] when client is not provided or its not expected class type
+      def cfn_resource(client)
+        fail ArgumentError,
+             'must be instance of Aws::CloudFormation::Client' unless client.instance_of?(Aws::CloudFormation::Client)
+        Aws::CloudFormation::Resource.new(client: client)
+      end
+    end
+  end
+end

data/lib/enscalator/helpers.rb

@@ -0,0 +1,127 @@
+require_relative 'helpers/sub_process'
+require_relative 'helpers/wrappers'
+require_relative 'helpers/stack'
+require_relative 'helpers/dns'
+
+# Enscalator
+module Enscalator
+  # Default directory to save generated assets like ssh keys, configs and etc.
+  ASSETS_DIR = File.join(ENV['HOME'], ".#{name.split('::').first.downcase}")
+
+  # Collection of helper classes and static methods
+  module Helpers
+    include Wrappers
+    include Stack
+    include Dns
+
+    # Initialize enscalator directory
+    # @return [String]
+    def init_assets_dir
+      FileUtils.mkdir_p(Enscalator::ASSETS_DIR) unless Dir.exist?(Enscalator::ASSETS_DIR)
+    end
+
+    # Provision Aws.config with custom settings
+    # @param [String] region valid aws region
+    # @param [String] profile_name aws credentials profile name
+    def init_aws_config(region, profile_name: nil)
+      fail ArgumentError, 'Unable to proceed without region' if region.blank?
+      opts = {}
+      opts[:region] = region
+      opts[:credentials] = Aws::SharedCredentials.new(profile_name: profile_name) unless profile_name.blank?
+      Aws.config.update(opts)
+    end
+
+    # Find ami images registered
+    #
+    # @param [Aws::EC2::Client] client instance of AWS EC2 client
+    # @return [Hash] images satisfying query conditions
+    # @raise [ArgumentError] when client is not provided or its not expected class type
+    def find_ami(client, owners: ['self'], filters: nil)
+      fail ArgumentError, 'must be instance of Aws::EC2::Client' unless client.instance_of?(Aws::EC2::Client)
+      query = {}
+      query[:dry_run] = false
+      query[:owners] = owners if owners.is_a?(Array) && owners.any?
+      query[:filters] = filters if filters.is_a?(Array) && filters.any?
+      client.describe_images(query)
+    end
+
+    # Generate ssh keyname from app_name, region and stack name
+    #
+    # @param [String] app_name application name
+    # @param [String] region aws region
+    # @param [String] stack_name cloudformation stack name
+    def gen_ssh_key_name(app_name, region, stack_name)
+      [app_name, region, stack_name].map(&:underscore).join('_')
+    end
+
+    # Create ssh public/private key pair, save private key for current user
+    #
+    # @param [String] key_name key name
+    # @param [String] region aws region
+    # @param [Boolean] force_create force to create a new ssh key
+    def create_ssh_key(key_name, region, force_create: false)
+      # Ignoring attempts to generate new ssh key when not deploying
+      if @options && @options[:expand]
+        warn '[Warning] SSH key can be generated only for create or update stack actions'
+        return
+      end
+
+      client = ec2_client(region)
+      aws_profile = if Aws.config.key?(:credentials)
+                      creds = Aws.config[:credentials]
+                      creds.profile_name if creds.respond_to?(:profile_name)
+                    end
+      target_dir = File.join(Enscalator::ASSETS_DIR, aws_profile ? aws_profile : 'default')
+      FileUtils.mkdir_p(target_dir) unless Dir.exist? target_dir
+      if !client.describe_key_pairs.key_pairs.collect(&:key_name).include?(key_name) || force_create
+        # delete existed ssh key
+        client.delete_key_pair(key_name: key_name)
+
+        # create a new ssh key
+        key_pair = client.create_key_pair(key_name: key_name)
+        STDERR.puts "Created new ssh key with fingerprint: #{key_pair.key_fingerprint}"
+
+        # save private key for current user
+        private_key = File.join(target_dir, key_name)
+        File.open(private_key, 'w') do |wfile|
+          wfile.write(key_pair.key_material)
+        end
+        STDERR.puts "Saved created key to: #{private_key}"
+        File.chmod(0600, private_key)
+      else
+        key_fingerprint =
+          begin
+            Aws::EC2::KeyPair.new(key_name, client: client).key_fingerprint
+          rescue NotImplementedError
+            # TODO: after upgrade of aws-sdk use only Aws::EC2::KeyPairInfo
+            Aws::EC2::KeyPairInfo.new(key_name, client: client).key_fingerprint
+          end
+        STDERR.puts "Found existing ssh key with fingerprint: #{key_fingerprint}"
+      end
+    end
+
+    # Read user data from file
+    #
+    # @param [String] app_name application name
+    def read_user_data(app_name)
+      user_data_path = File.join(File.expand_path('..', __FILE__), 'plugins', 'user-data', app_name)
+      fail("User data path #{user_data_path} not exists") unless File.exist?(user_data_path)
+      File.read(user_data_path)
+    end
+
+    # Convert hash with nested values to flat hash
+    #
+    # @param [Hash] input that should be flatten
+    def flatten_hash(input)
+      input.each_with_object({}) do |(k, v), h|
+        if v.is_a?(Hash)
+          flatten_hash(v).map do |h_k, h_v|
+            h["#{k}.#{h_k}".to_sym] = h_v
+          end
+        else
+          h[k] = v
+        end
+      end
+    end
+  end # module Helpers
+end # module Enscalator