enscalator 0.4.0.pre.alpha.pre.16

data/lib/enscalator/plugins/elasticsearch_amazon.rb

@@ -0,0 +1,75 @@
+module Enscalator
+  module Plugins
+    # Amazon Elasticsearch Service (Amazon ES)
+    module ElasticsearchAmazon
+      # Create new service instance
+      # @param [String] cluster_name name of the cluster resource
+      # @param [Hash] properties additional parameters for cluster configuration
+      def elasticsearch_init(cluster_name, properties: {})
+        cluster_properties = {
+          AccessPolicies: {
+            Version: '2012-10-17',
+            Statement: [
+              {
+                Effect: 'Allow',
+                Principal: {
+                  AWS: '*'
+                }
+              }
+            ]
+          },
+          AdvancedOptions: {
+            'rest.action.multi.allow_explicit_index': 'true'
+          },
+          EBSOptions: {
+            EBSEnabled: true,
+            Iops: 0,
+            VolumeSize: 100,
+            VolumeType: 'gp2'
+          },
+          ElasticsearchClusterConfig: {
+            InstanceCount: '1',
+            InstanceType: 'm3.medium.elasticsearch'
+          },
+          SnapshotOptions: {
+            AutomatedSnapshotStartHour: '0'
+          }
+        }
+
+        # do not modify properties passed from template
+        props = properties.deep_dup
+
+        default_tags = [
+          {
+            Key: 'ClusterName',
+            Value: cluster_name.downcase
+          }
+        ]
+
+        if props.key?(:Tags) && !props[:Tags].empty?
+          props[:Tags].concat(default_tags)
+        else
+          props[:Tags] = default_tags
+        end
+
+        resource cluster_name,
+                 Type: 'AWS::Elasticsearch::Domain',
+                 Properties: cluster_properties.merge(props)
+
+        output "#{cluster_name}ResourceID",
+               Description: "#{cluster_name} ResourceID",
+               Value: ref(cluster_name)
+
+        output "#{cluster_name}DomainArn",
+               Description: "#{cluster_name} DomainArn",
+               Value: get_att(cluster_name, 'DomainArn')
+
+        output "#{cluster_name}DomainEndpoint",
+               Description: "#{cluster_name} DomainEndpoint",
+               Value: get_att(cluster_name, 'DomainEndpoint')
+
+        cluster_name
+      end
+    end # module ElasticsearchAmazon
+  end # module Plugins
+end # module Enscalator

data/lib/enscalator/plugins/elasticsearch_bitnami.rb

@@ -0,0 +1,198 @@
+module Enscalator
+  module Plugins
+    # Elasticsearch related configuration
+    module ElasticsearchBitnami
+      # Retrieves mapping for Elasticsearch Bitnami stack
+      class << self
+        # Supported storage types in AWS
+        STORAGE = [:ebs, :'instance-store']
+
+        # Supported Elasticsearch image architectures
+        ARCH = [:amd64, :i386]
+
+        # Get ami region/virtualization type mapping
+        #
+        # @param [Symbol] storage image root storage
+        # @param [Symbol] arch image architecture type
+        # @return [Hash] mapping
+        def get_mapping(storage: :ebs, arch: :amd64)
+          fail ArgumentError, "storage can only be one of #{STORAGE}" unless STORAGE.include? storage
+          fail ArgumentError, "arch can only be one of #{ARCH}" unless ARCH.include? arch
+          fetch_mapping(storage, arch)
+        end
+
+        # Get ami release version string
+        #
+        # @param [Symbol] storage image root storage
+        # @param [Symbol] arch image architecture type
+        # @return [Hash] mapping
+        def get_release_version(storage: :ebs, arch: :amd64)
+          fail ArgumentError, "storage can only be one of #{STORAGE}" unless STORAGE.include? storage
+          fail ArgumentError, "arch can only be one of #{ARCH}" unless ARCH.include? arch
+          fetch_versions
+            .select { |r| r.root_storage == storage && r.arch == arch }
+            .map { |v| v.version.to_s }.uniq.first
+            .gsub(/[-][\w\d]/, '')
+        end
+
+        private
+
+        # Structure to hold parsed record
+        Struct.new('ElasticSearch', :name, :version, :baseos, :root_storage, :arch, :region, :ami, :virtualization)
+
+        # Always fetches the most recent version
+        #
+        # @param [Symbol] storage image root storage
+        # @param [Symbol] arch image architecture type
+        # @return [Hash] mapping
+        def fetch_mapping(storage, arch)
+          versions = fetch_versions
+          versions.select { |r| r.root_storage == storage && r.arch == arch }
+            .group_by(&:region)
+            .map { |k, v| [k, v.map { |i| [i.virtualization, i.ami] }.to_h] }.to_h
+            .with_indifferent_access
+        end
+
+        # Make request to Bitnami Elasticsearch release pages, parse response and make list of versions
+        #
+        # @return [Array] list of all versions across all AWS regions
+        def fetch_versions
+          html = Nokogiri::HTML(open('https://bitnami.com/stack/elasticsearch/cloud/amazon'))
+          raw_entries = html.xpath('//td[@class="instance_id"]')
+          entries = raw_entries.xpath('a')
+          raw_entries.xpath('strong/a').each { |sa| entries << sa }
+          raw_versions = entries.map do |i|
+            [
+              i.xpath('@href').first.value.split('/').last,
+              i.children.first.text
+            ]
+          end.to_h
+          parse_versions(raw_versions)
+        end
+
+        # Parse list of raw strings
+        #
+        # @param entries [Array] list of strings
+        # @return [Array]
+        def parse_versions(entries)
+          entries.map do |rw, ami|
+            str, region = rw.split('?').map { |s| s.start_with?('region') ? s.split('=').last : s }
+            version_str = fix_entry(str).split('-')
+            name, version, baseos = version_str
+            Struct::ElasticSearch.new(name,
+                                      Semantic::Version.new(version.tr('=', '-')),
+                                      baseos,
+                                      version_str.include?('ebs') ? :ebs : :'instance-store',
+                                      version_str.include?('x64') ? :amd64 : :i386,
+                                      region,
+                                      ami,
+                                      version_str.include?('hvm') ? :hvm : :pv)
+          end
+        end
+
+        # Fix elasticsearch version string to have predictable format
+        #
+        # @param [String] str raw version string
+        # @return [String] reformatted version string
+        def fix_entry(str)
+          pattern = '[-](?:[\w\d]+){1,3}[-]ami'
+          token = begin
+            Regexp.new(pattern.gsub('?:', '')).match(str)[1]
+          rescue
+            nil
+          end
+          str.gsub(Regexp.new(pattern), ['=', token, '-'].join)
+        end
+      end # class << self
+
+      # Get aws account id
+      # @return [String] account id
+      def aws_account_id
+        # noinspection RubyArgCount
+        Aws::IAM::Client.new.get_user.user.arn.split(':')[4]
+      end
+
+      # Create new elasticsearch instance
+      #
+      # @param [String] storage_name storage name
+      # @param [Integer] allocated_storage size of instance primary storage
+      # @param [String] instance_type instance type
+      # @param [Hash] properties additional properties
+      # @param [String] zone_name route53 zone name
+      def elasticsearch_init(storage_name,
+                             allocated_storage: 5,
+                             instance_type: 't2.medium',
+                             properties: {},
+                             zone_name: nil)
+
+        @es_key_name = gen_ssh_key_name("Elasticsearch#{storage_name}", region, stack_name)
+        pre_run { create_ssh_key(@es_key_name, region, force_create: false) }
+
+        mapping 'AWSElasticsearchAMI', ElasticsearchBitnami.get_mapping
+
+        parameter_allocated_storage "Elasticsearch#{storage_name}",
+                                    default: allocated_storage,
+                                    min: 5,
+                                    max: 1024
+
+        parameter_ec2_instance_type "Elasticsearch#{storage_name}", type: instance_type
+
+        properties[:KeyName] = @es_key_name
+        properties[:InstanceType] = ref("Elasticsearch#{storage_name}InstanceType")
+
+        version_tag = {
+          Key: 'Version',
+          Value: ElasticsearchBitnami.get_release_version
+        }
+
+        cluster_name_tag = {
+          Key: 'ClusterName',
+          Value: storage_name.downcase
+        }
+
+        plugin_tags = [version_tag, cluster_name_tag]
+
+        # Set instance tags
+        if properties.key?(:Tags) && !properties[:Tags].empty?
+          properties[:Tags].concat(plugin_tags)
+        else
+          properties[:Tags] = plugin_tags
+        end
+
+        # Configure instance using user-data
+        if !properties.key?(:UserData) || !properties[:UserData].empty?
+          properties[:UserData] = Base64.encode64(read_user_data('elasticsearch'))
+        end
+
+        # Assign IAM role to instance
+        properties[:IamInstanceProfile] = iam_instance_profile_with_full_access(storage_name, *%w(ec2 s3))
+
+        storage_resource_name = "Elasticsearch#{storage_name}"
+        instance_vpc storage_resource_name,
+                     find_in_map('AWSElasticsearchAMI', ref('AWS::Region'), :hvm),
+                     ref_application_subnets.first,
+                     [ref_private_security_group, ref_resource_security_group],
+                     depends_on: [],
+                     properties: properties
+
+        # create s3 bucket for cluster snapshots
+        account_id = aws_account_id
+        bucket_name = "elasticsearch-bitnami-#{region}-#{account_id}"
+        resource "Elasticsearch#{storage_name}S3Bucket",
+                 Type: 'AWS::S3::Bucket',
+                 DeletionPolicy: 'Retain',
+                 Properties: {
+                   BucketName: bucket_name
+                 }
+
+        # create a DNS record in route53 for instance private ip
+        record_name = %W(#{storage_name.downcase.dasherize} #{region} #{zone_name}).join('.')
+        create_single_dns_record("#{storage_name}PrivateZone",
+                                 stack_name,
+                                 zone_name,
+                                 record_name,
+                                 resource_records: [get_att(storage_resource_name, 'PrivateIp')])
+      end
+    end # module ElasticsearchBitnami
+  end # module Plugins
+end # module Enscalator

data/lib/enscalator/plugins/elasticsearch_opsworks.rb

@@ -0,0 +1,225 @@
+module Enscalator
+  module Plugins
+    # Elasticsearch related configuration
+    module ElasticsearchOpsworks
+      include Enscalator::Plugins::Elb
+
+      # Create Elasticsearch cluster using Opsworks
+      #
+      # @param [String] app_name application name
+      # @param [String] ssh_key name of the ssh key
+      # @param [String] os base operating system
+      # @param [String] cookbook chef cookbook
+      def elasticsearch_init(app_name,
+                             ssh_key:,
+                             es_config: {},
+                             os: 'Amazon Linux 2015.09',
+                             cookbook: 'https://github.com/en-japan-air/opsworks-elasticsearch-cookbook.git')
+
+        parameter "ES#{app_name}ChefCookbook",
+                  Default: cookbook,
+                  Description: 'GitURL',
+                  Type: 'String'
+
+        parameter "ES#{app_name}InstanceDefaultOs",
+                  Default: os,
+                  Description: ['The stack\'s default operating system, which is installed',
+                                'on every instance unless you specify a different',
+                                'operating system when you create the instance.'].join(' '),
+                  Type: 'String'
+
+        parameter "ES#{app_name}SshKeyName",
+                  Default: ssh_key,
+                  Description: 'SSH key name for EC2 instances.',
+                  Type: 'String'
+
+        resource 'InstanceRole',
+                 Type: 'AWS::IAM::InstanceProfile',
+                 Properties: {
+                   Path: '/',
+                   Roles: [
+                     ref('OpsWorksEC2Role')
+                   ]
+                 }
+
+        resource 'ServiceRole',
+                 Type: 'AWS::IAM::Role',
+                 Properties: {
+                   AssumeRolePolicyDocument: {
+                     Statement: [
+                       {
+                         Effect: 'Allow',
+                         Principal: {
+                           Service: [
+                             'opsworks.amazonaws.com'
+                           ]
+                         },
+                         Action: [
+                           'sts:AssumeRole'
+                         ]
+                       }
+                     ]
+                   },
+                   Path: '/',
+                   Policies: [
+                     {
+                       PolicyName: "#{app_name}-opsworks-service",
+                       PolicyDocument: {
+                         Statement: [
+                           {
+                             Effect: 'Allow',
+                             Action: %w( ec2:* iam:PassRole cloudwatch:GetMetricStatistics elasticloadbalancing:* ),
+                             Resource: '*'
+                           }
+                         ]
+                       }
+                     }
+                   ]
+                 }
+
+        resource 'OpsWorksEC2Role',
+                 Type: 'AWS::IAM::Role',
+                 Properties: {
+                   AssumeRolePolicyDocument: {
+                     Statement: [
+                       {
+                         Effect: 'Allow',
+                         Principal: {
+                           Service: [
+                             'ec2.amazonaws.com'
+                           ]
+                         },
+                         Action: [
+                           'sts:AssumeRole'
+                         ]
+                       }
+                     ]
+                   },
+                   Path: '/',
+                   Policies: [
+                     {
+                       PolicyName: "#{app_name}-opsworks-ec2-role",
+                       PolicyDocument: {
+                         Statement: [
+                           {
+                             Effect: 'Allow',
+                             Action: %w(
+                               ec2:DescribeInstances
+                               ec2:DescribeRegions
+                               ec2:DescribeSecurityGroups
+                               ec2:DescribeTags
+                               cloudwatch:PutMetricData),
+                             Resource: '*'
+                           }
+                         ]
+                       }
+                     }
+                   ]
+                 }
+
+        instances_security_group = security_group_vpc("ES#{app_name}", 'so that ES cluster can find other nodes', vpc.id)
+
+        ops_stack_name = "#{app_name}-ES"
+        resource 'ESStack',
+                 Type: 'AWS::OpsWorks::Stack',
+                 Properties: {
+                   Name: ops_stack_name,
+                   VpcId: vpc.id,
+                   DefaultSubnetId: ref_resource_subnets.first,
+                   ConfigurationManager: {
+                     Name: 'Chef',
+                     Version: '12'
+                   },
+                   UseCustomCookbooks: 'true',
+                   CustomCookbooksSource: {
+                     Type: 'git',
+                     Url: ref("ES#{app_name}ChefCookbook")
+                   },
+                   DefaultOs: ref("ES#{app_name}InstanceDefaultOs"),
+                   DefaultRootDeviceType: 'ebs',
+                   DefaultSshKeyName: ref("ES#{app_name}SshKeyName"),
+                   CustomJson: {
+                     java: {
+                       jdk_version: '8',
+                       oracle: {
+                         accept_oracle_download_terms: 'true'
+                       },
+                       accept_license_agreement: 'true',
+                       install_flavor: 'oracle'
+                     },
+                     elasticsearch: {
+                       plugins: [
+                         'analysis-kuromoji',
+                         'cloud-aws',
+                         { name: 'elasticsearch-head', url: 'mobz/elasticsearch-head' }
+                       ],
+                       config: {
+                         'cluster.name': "#{app_name}-elasticsearch",
+                         'path.data': '/mnt/elasticsearch-data',
+                         'path.logs': '/mnt/elasticsearch-data/logs/',
+                         'network.bind_host': '0.0.0.0',
+                         'network.publish_host': '_non_loopback_',
+                         'index.routing.allocation.disable_allocation': 'false',
+                         'cloud.aws.region': region,
+                         discovery: {
+                           type: 'ec2',
+                           ec2: {
+                             groups: [ref(instances_security_group)],
+                             tag: {
+                               'opsworks:stack': ops_stack_name
+                             }
+                           }
+                         },
+                         'cluster.routing.allocation.awareness.attributes': 'rack_id'
+                       }.merge(es_config)
+                     }
+                   },
+                   ServiceRoleArn: {
+                     'Fn::GetAtt': %w(ServiceRole Arn)
+                   },
+                   DefaultInstanceProfileArn: {
+                     'Fn::GetAtt': %w(InstanceRole Arn)
+                   }
+                 }
+
+        resource 'ESLayer',
+                 Type: 'AWS::OpsWorks::Layer',
+                 Properties: {
+                   StackId: ref('ESStack'),
+                   Name: 'Search',
+                   Type: 'custom',
+                   Shortname: 'search',
+                   CustomRecipes: {
+                     Setup: %w(apt ark java layer-custom::es-opsworks)
+                   },
+                   EnableAutoHealing: 'true',
+                   AutoAssignElasticIps: 'false',
+                   AutoAssignPublicIps: 'false',
+                   VolumeConfigurations: [
+                     {
+                       MountPoint: '/mnt/elasticsearch-data',
+                       NumberOfDisks: 1,
+                       VolumeType: 'gp2',
+                       Size: 100
+                     }
+                   ],
+                   CustomSecurityGroupIds: [
+                     {
+                       'Fn::GetAtt': %W(#{instances_security_group} GroupId)
+                     },
+                     ref_private_security_group
+                   ]
+                 }
+
+        resource 'ESMainInstance',
+                 Type: 'AWS::OpsWorks::Instance',
+                 Properties: {
+                   # EbsOptimized: true,          # Not available for m3.medium
+                   InstanceType: 'm3.medium',
+                   LayerIds: [ref('ESLayer')],
+                   StackId: ref('ESStack')
+                 }
+      end
+    end # module Elasticsearch
+  end # module Plugins
+end # module Enscalator

data/lib/enscalator/plugins/elb.rb

@@ -0,0 +1,139 @@
+module Enscalator
+  module Plugins
+    # Internet facing ELB instance
+    module Elb
+      # Create new ELB instance
+      #
+      # @param [String, Hash] elb_name ELB instance name - can be either String or Fn::Join
+      # @param [Integer] web_server_port application port to which ELB redirects traffic
+      # @param [String] zone_name zone name attached to the vpc
+      # @return [String] ELB resource name
+      def elb_init(elb_name: join('-', aws_stack_name, 'elb'),
+                   web_server_port: 9000,
+                   health_check_path: '/',
+                   zone_name: nil,
+                   dns_record_name: "elb.#{stack_name.dasherize}.#{zone_name}",
+                   instances: [],
+                   ssl: false,
+                   internal: true)
+
+        elb_resource_name = 'LoadBalancer'
+
+        parameter 'WebServerPort',
+                  Description: 'TCP/IP Port for the web service',
+                  Default: web_server_port,
+                  Type: 'Number',
+                  MinValue: '0',
+                  MaxValue: '65535',
+                  ConstraintDescription: 'must be an integer between 0 and 65535.'
+
+        security_group_vpc 'ELBSecurityGroup',
+                           'Security group of the application servers',
+                           ref('VpcId'),
+                           security_group_ingress: [
+                             {
+                               IpProtocol: 'tcp',
+                               FromPort: '0',
+                               ToPort: '65535',
+                               CidrIp: '10.0.0.0/8'
+                             }
+                           ] + (internal ? [] : [
+                             {
+                               IpProtocol: 'tcp',
+                               FromPort: '80',
+                               ToPort: '80',
+                               CidrIp: '0.0.0.0/0'
+                             },
+                             {
+                               IpProtocol: 'tcp',
+                               FromPort: '443',
+                               ToPort: '443',
+                               CidrIp: '0.0.0.0/0'
+                             },
+                             {
+                               IpProtocol: 'tcp',
+                               FromPort: '465',
+                               ToPort: '465',
+                               CidrIp: '0.0.0.0/0'
+                             }
+                           ]),
+                           tags: {
+                             Name: join('-', aws_stack_name, 'app', 'sg'),
+                             Application: aws_stack_name
+                           }
+
+        if ssl
+          parameter 'SSLCertificateId',
+                    Description: 'Id of the SSL certificate (iam-servercertgetattributes -s certname)',
+                    Type: 'String',
+                    ConstraintDescription: 'must be a string'
+        end
+
+        properties = {
+          LoadBalancerName: elb_name,
+          Listeners: [
+            {
+              LoadBalancerPort: '80',
+              InstancePort: ref('WebServerPort'),
+              Protocol: 'HTTP'
+            }
+          ] + (ssl == false ? [] : [
+            { LoadBalancerPort: '443',
+              InstancePort: ref('WebServerPort'),
+              SSLCertificateId: ref('SSLCertificateId'),
+              Protocol: 'HTTPS' }
+          ]),
+          HealthCheck: {
+            Target: join('', 'HTTP:', ref_web_server_port, health_check_path),
+            HealthyThreshold: '3',
+            UnhealthyThreshold: '5',
+            Interval: '30',
+            Timeout: '5'
+          },
+          SecurityGroups: [ref('ELBSecurityGroup')],
+          Subnets: internal ? ref_application_subnets : public_subnets,
+          Tags: [
+            {
+              Key: 'Name',
+              Value: elb_name
+            },
+            {
+              Key: 'Application',
+              Value: aws_stack_name
+            },
+            {
+              Key: 'Network',
+              Value: (internal ? 'Private' : 'Public')
+            }
+          ]
+        }
+
+        properties[:Scheme] = 'internal' if internal
+        properties[:Instances] = instances if instances && !instances.empty?
+
+        resource elb_resource_name,
+                 Type: 'AWS::ElasticLoadBalancing::LoadBalancer',
+                 Properties: properties
+
+        # use alias target to create proper cloudformation template for Route53 side of elb configuration
+        alias_target = {
+          HostedZoneId: get_att(elb_resource_name, 'CanonicalHostedZoneNameID'),
+          DNSName: get_att(elb_resource_name, internal ? 'DNSName' : 'CanonicalHostedZoneName')
+        }
+
+        create_single_dns_record(nil,
+                                 stack_name,
+                                 zone_name,
+                                 dns_record_name,
+                                 alias_target: alias_target)
+
+        output "#{elb_resource_name}DNSName",
+               Description: 'LoadBalancer DNS Name',
+               Value: get_att(elb_resource_name, 'DNSName')
+
+        # return resource name
+        elb_resource_name
+      end
+    end # module Elb
+  end # module Plugins
+end # module Enscalator