enscalator 0.4.0.pre.alpha.pre.16

data/lib/enscalator/plugins/user-data/elasticsearch

@@ -0,0 +1,367 @@
+#cloud-config
+
+packages:
+  - ruby
+
+bootcmd:
+  - update-rc.d monit disable
+  - update-rc.d bitnami disable
+
+runcmd:
+  - gem install aws-sdk semantic backports elasticsearch --no-ri --no-rdoc
+  - . /usr/local/bin/configure-boot-env
+  - /etc/init.d/monit stop
+  - /opt/bitnami/ctlscript.sh stop apache
+  - /opt/bitnami/ctlscript.sh stop elasticsearch
+  - sleep 5
+  - kill $(pgrep -f elasticsearch)
+  - /usr/local/bin/configure-cluster
+  - /etc/init.d/monit start
+  - /opt/bitnami/ctlscript.sh start apache
+  - /opt/bitnami/ctlscript.sh start elasticsearch
+  - echo "Verifying if hourly cronjob was installed correctly.." && run-parts --test /etc/cron.hourly
+  - /usr/local/bin/backup-cluster --restore-snapshot
+
+write_files:
+  - path: /usr/local/bin/configure-boot-env
+    owner: root:root
+    permissions: '0755'
+    content: |
+      #!/usr/bin/env bash
+      #
+      # Configure boot environment to properly detect bitnami stack
+
+      BITNAMI_DIR=/opt/bitnami
+      ELASTICSEARCH_DIR=$BITNAMI_DIR/elasticsearch
+      APACHE2_DIR=$BITNAMI_DIR/apache2
+      BITNAMI_COMMON_DIR=$BITNAMI_DIR/common
+      JAVA_DIR=$BITNAMI_DIR/java
+      SYSTEM_PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
+      export PATH=$ELASTICSEARCH_DIR/bin:$APACHE2_DIR/bin:$BITNAMI_COMMON_DIR/bin:$JAVA_DIR/bin:$SYSTEM_PATH
+      export JAVA_HOME=$JAVA_DIR
+
+  - path: /usr/local/bin/configure-cluster
+    owner: root:root
+    permissions: '0755'
+    content: |
+      #!/usr/bin/env ruby
+      #
+      # Elasticsearch configuration
+      #
+      # PURPOSE: update elasticsearch configuration using values from instance tags
+      #
+      # WARNING: since this script also modifies ip address settings, make sure
+      # to stop elasticsearch BEFORE using this script
+
+      require 'yaml'
+      require 'open-uri'
+      require 'json'
+      require 'ostruct'
+      require 'fileutils'
+      require 'aws-sdk'
+      require 'semantic'
+      require 'backports'
+
+      BITNAMI_INSTALL_PATH='/opt/bitnami'
+      ELASTICSEARCH_INSTALL_PATH=File.join(BITNAMI_INSTALL_PATH, 'elasticsearch')
+      ELASTICSEARCH_CONFIG_FILE=File.join(ELASTICSEARCH_INSTALL_PATH, 'config', 'elasticsearch.yml')
+
+      class Plugin
+        attr_reader :install_prefix, :install_link, :mapping, :es_version
+        def initialize(install_prefix, link, mapping, es_version)
+          @install_prefix ||= install_prefix
+          @install_link ||= link
+          @mapping ||= mapping.map {|k,v| [Semantic::Version.new(k.to_s), v] }.to_h
+          @es_version ||= Semantic::Version.new(es_version)
+        end
+
+        def find_plugin_version
+          @mapping[@mapping.keys.select { |v| v.major == @es_version.major && v.minor == @es_version.minor }.first]
+        end
+
+        def install
+          path = [@install_link, find_plugin_version].join('/')
+          cmd = [File.join(@install_prefix, 'bin/plugin'), 'install', path].join(' ')
+          IO.popen(cmd) { |io| io.read }
+        end
+      end
+
+      document = 'http://169.254.169.254/latest/dynamic/instance-identity/document/'
+      metadata = OpenStruct.new(JSON.parse(open(document){|d| d.read }))
+      ec2 = Aws::EC2::Client.new(region: metadata.region)
+
+      tags = nil
+      ec2.describe_tags.each do |response|
+        tags = response[:tags].select { |t| t.resource_id == metadata.instanceId && t.resource_type == 'instance' }
+      end
+
+      cluster_name = tags.select { |t| t.key == 'ClusterName' }.first.value
+      cluster_version = tags.select { |t| t.key == 'Version' }.first.value
+
+      # generate elasticsearch configuration using instance metadata
+      config = {}
+      config['cluster.name'] = cluster_name
+      config['network.host'] = metadata.privateIp
+      config['http.port'] = 9200
+      config['transport.tcp.port'] = 9300
+      config['node.max_local_storage_nodes'] = 1
+      config['plugin.mandatory'] = 'cloud-aws'
+      config['discovery.type'] = 'ec2'
+      config['discovery.zen.ping.multicast.enabled'] = false
+      config['discovery.ec2.tag.ClusterName'] = cluster_name
+      config['discovery.ec2.ping_timeout'] = '20s'
+
+      # if configuration file already present, create its backup and remove original file
+      if File.exists?(ELASTICSEARCH_CONFIG_FILE)
+        FileUtils.move(ELASTICSEARCH_CONFIG_FILE, [ELASTICSEARCH_CONFIG_FILE, 'bak'].join('.'))
+      end
+
+      # write generated configuration file
+      File.open(ELASTICSEARCH_CONFIG_FILE, 'w+') { |file| file.write(config.to_yaml) }
+
+      # install necessary plugins
+      aws = Plugin.new(ELASTICSEARCH_INSTALL_PATH, 'cloud-aws', {}, cluster_version)
+      kuromoji = Plugin.new(ELASTICSEARCH_INSTALL_PATH, 'analysis-kuromoji', {}, cluster_version)
+
+      # execute plugin install command
+      [aws, kuromoji].map(&:install).each { |r| puts r }
+
+  - path: /usr/local/bin/backup-cluster
+    owner: root:root
+    permissions: '0755'
+    content: |
+      #!/usr/bin/env ruby
+
+      require 'elasticsearch'
+      require 'optparse'
+      require 'ostruct'
+      require 'open-uri'
+      require 'json'
+      require 'backports'
+
+      module Storage
+        class Backup
+          attr_reader :instance, :repository, :config
+
+          def initialize(host, opts = {})
+            defaults = {
+                port: 9200,
+                repository: 'backup',
+                config: nil
+            }
+            opts = defaults.merge(opts)
+
+            @instance = ESInstance.new(host, opts[:port])
+            @instance.wait_for_cluster_to_start
+            @config = opts[:config].nil? || opts[:config].empty? ? s3_config : opts[:config]
+            @repository = opts[:repository]
+          end
+
+          def fs_config
+            {
+                type: 'fs',
+                settings: {
+                    location: '/tmp/backup',
+                    compress: true
+                }
+            }
+          end
+
+          def instance_metadata
+            document = 'http://169.254.169.254/latest/dynamic/instance-identity/document/'
+            OpenStruct.new(JSON.parse(open(document){|d| d.read }))
+          end
+
+          # TODO: adjust configuration to be more flexible
+          def s3_config
+            metadata = instance_metadata
+            path_in_bucket = get_cluster_name # for now use cluster name
+            {
+                type: 's3',
+                settings: {
+                    bucket: "elasticsearch-bitnami-#{metadata.region}-#{metadata.accountId}",
+                    region: metadata.region,
+                    base_path: path_in_bucket
+                }
+            }
+          end
+
+          # Create/Init snapshot repository
+          def create_repository
+            @instance.client
+                .snapshot
+                .create_repository({repository: @repository,
+                                    body: @config})
+          end
+
+          # Get list of registered snapshots
+          def get_all_snapshots
+            raw_snapshots = @instance.client
+                                .snapshot
+                                .get(repository: @repository,
+                                     snapshot: '_all')
+            raw_snapshots['snapshots'].map { |rs| OpenStruct.new(rs) }
+          end
+
+          # Determine last snapshot in the list and invoke restore call with it
+          # TODO: algorithm to determine which snapshot to use, should take into account failures
+          # TODO: provide ability to override/use specific snapshot when instance gets created
+          def restore_from_last_snapshot
+            snapshots = get_all_snapshots.select { |s| s.state.eql? 'SUCCESS' }
+            last = snapshots.sort_by { |x| [x.start_time_in_millis, x.end_time_in_millis] }.last
+            if last
+              restore_snapshot(last)
+            else
+              puts 'Failed to find a list of valid snapshots for the cluster'
+            end
+          end
+
+          def get_cluster_name
+            @instance.client.cluster.state['cluster_name']
+          end
+
+          # Create new snapshot
+          # snapshot pattern: %{cluster_name}_%{increment}
+          def take_snapshot(opts = {})
+            defaults = {
+                create_opts: nil,
+                master_timeout: nil,
+                wait_for_completion: false
+            }
+            defaults.merge(opts)
+
+            cluster_name = get_cluster_name
+            registered_snapshots = get_all_snapshots.map(&:snapshot)
+            default_index = 1
+            index = if !registered_snapshots.empty?
+                      last_index = registered_snapshots
+                                       .map { |s| s.split('_').last }
+                                       .select { |s| s =~ /^[0-9][0-9]*/ }.sort.last
+                      last_index.to_i + 1
+                    else
+                      default_index
+                    end
+            options = {}
+            options[:body] = opts[:create_opts] if opts[:create_opts].instance_of?(Hash) && !opts[:create_opts].empty?
+            options[:repository] = @repository
+            options[:snapshot] = [cluster_name, index].join('_')
+            options[:master_timeout] = opts[:master_timeout] if opts[:master_timeout]
+            options[:wait_for_completion] = opts[:wait_for_completion]
+            @instance.client.snapshot.create(options)
+          end
+
+          # Restore given snapshot
+          def restore_snapshot(snapshot, opts = {})
+            raise ArgumentError, 'instance with snapshot fields is required' unless (snapshot.instance_of?(OpenStruct) || snapshot.snapshot.empty?)
+            defaults = {
+                restore_opts: nil,
+                master_timeout: nil,
+                wait_for_completion: false
+            }
+
+            opts = defaults.merge(opts)
+
+            options = {}
+            options[:repository] = @repository
+            options[:snapshot] = snapshot.snapshot
+            options[:body] = opts[:restore_opts] if opts[:restore_opts].instance_of?(Hash) && !opts[:restore_opts].empty?
+            options[:master_timeout] = opts[:master_timeout] if opts[:master_timeout]
+            options[:wait_for_completion] = opts[:wait_for_completion]
+            @instance.client.snapshot.restore(options)
+          end
+
+        end
+
+        # Create elasticsearch instance reference and setup elasticsearch client
+        class ESInstance
+          attr_reader :host, :port, :client
+
+          def initialize(host, port)
+            @host = host
+            @port = port
+            @client = es_client(host, port)
+          end
+
+          def es_client(host, port, opts = {})
+            defaults = {
+                debug: true
+            }
+            opts = defaults.merge(opts)
+            Elasticsearch::Client.new({host: "#{host}:#{port}", log: opts[:debug]})
+          end
+
+          # Wait for cluster to become alive and ready to accept writes
+          def wait_for_cluster_to_start(retries = 10)
+
+            # check until cluster start to receive web requests
+            health =
+                begin
+                  @client.cluster.health
+                rescue
+                  sleep 5
+                  retry
+                end
+
+            if health['status'] == 'red'
+              sleep 5
+              retries == 0 && puts('Reached limit of retries')
+              retries > 0 && wait_for_cluster_to_start(retries - 1)
+            else
+              puts 'Cluster is ready to accept requests'
+            end
+
+          end
+
+        end
+
+      end
+
+      # Using built-in command-line options parser to reduce dependencies
+      options = {}
+      opt_parser =OptionParser.new do |opts|
+
+        opts.on('-c', '--create-snapshot', 'Create elasticsearch snapshot') do |cs|
+          options[:create_snapshot] = cs
+        end
+        opts.on('-r', '--restore-snapshot', 'Restore elasticsearch snapshot') do |rs|
+          options[:restore_snapshot] = rs
+        end
+
+      end
+
+      begin
+        opt_parser.parse!
+
+        document = 'http://169.254.169.254/latest/dynamic/instance-identity/document/'
+        metadata = OpenStruct.new(JSON.parse(open(document){|d| d.read }))
+        backup = Storage::Backup.new(metadata.privateIp)
+
+        if options[:create_snapshot]
+          puts 'Creating new snapshot'
+          backup.create_repository
+          backup.take_snapshot(master_timeout: 60)
+        end
+
+        if options[:restore_snapshot]
+          puts 'Trying to restore cluster state from snapshot'
+          backup.create_repository
+          backup.restore_from_last_snapshot(master_timeout: 60)
+        end
+
+        if options.empty?
+          puts opt_parser
+          exit 1
+        end
+
+      rescue RuntimeError => e
+        puts e
+      end
+
+  - path: /etc/cron.hourly/take_cluster_snapshot
+    owner: root:root
+    permissions: '0755'
+    content: |
+      #!/usr/bin/env bash
+
+      /usr/local/bin/backup-cluster --create-snapshot

data/lib/enscalator/plugins/vpc_peering_connection.rb

@@ -0,0 +1,48 @@
+module Enscalator
+  module Plugins
+    # VPC Peering Connection Plugin
+    module VPCPeeringConnection
+      # Template parameter for VPC ID
+      #
+      # @param [String] name parameter name
+      # @param [String] description parameter description
+      def parameter_vpc_id(name, description, default_value = nil)
+        options = {
+          Description: description,
+          Type: 'String',
+          AllowedPattern: 'vpc-[a-zA-Z0-9]*',
+          ConstraintDescription: 'must be valid VPC id (vpc-*).'
+        }
+        options[:Default] = default_value if default_value && !default_value.nil?
+        parameter name, options
+      end
+
+      # Create new vpc peering connection
+      #
+      # @param [String] conn_name connection name
+      # @param [Array<String>] tags list of tags
+      def vpc_peering_init(conn_name, tags: [])
+        options = {}
+        options[:Properties] = {
+          VpcId: ref("#{conn_name}VpcId"),
+          PeerVpcId: ref("#{conn_name}PeerVpcId")
+        }
+
+        # Set plugin tags
+        options[:Properties][:Tags] = tags if tags && !tags.empty?
+
+        resource conn_name,
+                 {
+                   Type: 'AWS::EC2::VPCPeeringConnection'
+                 }.merge(options)
+
+        output conn_name,
+               Description: 'VPC Peering connection name',
+               Value: ref(conn_name)
+
+        # return resource name
+        conn_name
+      end
+    end # module VPCPeeringConnection
+  end # module Plugins
+end # module Enscalator

data/lib/enscalator/plugins.rb

@@ -0,0 +1,30 @@
+require 'shellwords'
+require 'open-uri'
+require 'nokogiri'
+require 'semantic'
+
+require_relative 'plugins/route53'
+require_relative 'plugins/core_os'
+require_relative 'plugins/elb'
+require_relative 'plugins/auto_scale'
+require_relative 'plugins/couchbase'
+require_relative 'plugins/core_os'
+require_relative 'plugins/ubuntu'
+require_relative 'plugins/redis'
+require_relative 'plugins/debian'
+require_relative 'plugins/rethink_db'
+require_relative 'plugins/rds'
+require_relative 'plugins/elastic_beanstalk'
+require_relative 'plugins/elasticache'
+require_relative 'plugins/elasticsearch_amazon'
+require_relative 'plugins/elasticsearch_bitnami'
+require_relative 'plugins/elasticsearch_opsworks'
+require_relative 'plugins/amazon_linux'
+require_relative 'plugins/vpc_peering_connection'
+require_relative 'plugins/nat_gateway'
+
+module Enscalator
+  # Namespace for enscalator plugins
+  module Plugins
+  end
+end

data/lib/enscalator/rich_template_dsl.rb

@@ -0,0 +1,209 @@
+require 'cloudformation-ruby-dsl/cfntemplate'
+
+module Enscalator
+  # DSL specific for enscalator
+  class RichTemplateDSL < TemplateDSL
+    include Enscalator::Core::CfParameters
+    include Enscalator::Core::CfResources
+    include Enscalator::Helpers
+    include Enscalator::Plugins::Route53
+
+    # Cloudformation limit when sending template body directly
+    TEMPLATE_BODY_LIMIT = 51_200
+
+    # Create new RichTemplateDSL instance
+    #
+    # @param [Hash] options command-line arguments
+    def initialize(options = {})
+      @options = options
+      init_aws_config(@options[:region], profile_name: @options[:profile])
+      super(parse_params(@options[:parameters]), @options[:stack_name], @options[:region], false, &proc { tpl })
+    end
+
+    # Parse semicolon separated parameter string
+    #
+    # @param [String] raw_parameters raw parameter string
+    # @return [Hash] parameter hash
+    def parse_params(raw_parameters)
+      Hash[(raw_parameters || '').split(/;/).map { |pair| pair.split(/=/, 2) }]
+    end
+
+    # Helper method to check if the current command is to create the stack
+    #
+    # @return [Truthy] truthful if currently creating a stack
+    def creating?
+      @options[:create_stack]
+    end
+
+    # Helper method to provide accessor for `region`
+    #
+    # @return [String] region
+    def region
+      aws_region
+    end
+
+    # Helper method to provide value accessor for `vpc_stack_name`
+    #
+    # @return [String] vpc_stack_name
+    # @raise [RuntimeError] if vpc-stack-name was not given
+    def vpc_stack_name
+      @options[:vpc_stack_name] || fail('Requires vpc-stack-name')
+    end
+
+    # Adds trailing dot to make it proper FQDN
+    #
+    # @param [String] str fqdn string
+    # @return [String] fqdn with trailing dot
+    def handle_trailing_dot(str)
+      str.end_with?('.') ? str : str + '.'
+    end
+
+    # @deprecated
+    # Hosted zone accessor
+    #
+    # @return [String] hosted zone, and ensure ending with a '.'
+    # @raise [RuntimeError] if hosted zone is accessed but it's not configured
+    def hosted_zone
+      ActiveSupport::Deprecation.warn('hosted_zone is deprecated (use private_hosted_zone instead)')
+      private_hosted_zone
+    end
+
+    # Private hosted zone accessor
+    #
+    # @return [String] private hosted zone
+    # @raise [RuntimeError] if private hosted zone was accessed before it was configured
+    def private_hosted_zone
+      # TODO: adjust other templates/plugins to use private_hosted_zone
+      @options[:private_hosted_zone] || fail('Private hosted zone has to be configured')
+      handle_trailing_dot(@options[:private_hosted_zone])
+    end
+
+    # Public hosted zone accessor
+    #
+    # @return [String] public hosted zone
+    # @raise [RuntimeError] if hosted zone was accessed before it was configured
+    def public_hosted_zone
+      @options[:public_hosted_zone] || fail('Public hosted zone has to be configured')
+      handle_trailing_dot(@options[:public_hosted_zone])
+    end
+
+    # Get a list of availability zones for the given region
+    def read_availability_zones
+      az = @options[:availability_zone].to_sym
+      supported_az = ec2_client(region).describe_availability_zones.availability_zones
+      alive_az = supported_az.select { |zone| zone.state == 'available' }
+      az_list = alive_az.collect(&:zone_name).map { |n| [n.last.to_sym, n] }.to_h
+
+      # use all zones, specific one, or fail if zone is not supported in given region
+      if az.equal?(:all)
+        az_list
+      elsif az_list.keys.include?(az.to_sym)
+        az_list.select { |k, _| k == az }
+      else
+        fail("Requested zone #{az} is not supported in #{region}, supported ones are #{az_list.keys.join(',')}")
+      end
+    end
+
+    # Availability zones accessor
+    def availability_zones
+      @availability_zones ||= read_availability_zones
+    end
+
+    # Pre-run hook
+    #
+    # @param [Proc] block hook body
+    def pre_run(&block)
+      (@pre_run_blocks ||= []) << block if block_given?
+    end
+
+    # Post-run hook
+    #
+    # @param [Proc] block hook body
+    def post_run(&block)
+      (@post_run_blocks ||= []) << block if block_given?
+    end
+
+    # Convert tags to properties
+    #
+    # @param [Hash] tags collection of tags
+    # @return [Array] list of properties
+    def tags_to_properties(tags)
+      tags.map { |k, v| { Key: k, Value: v } }
+    end
+
+    # Template description
+    #
+    # @param [String] desc template description
+    def description(desc)
+      value(Description: desc)
+    end
+
+    # Network interface
+    #
+    # @param [String] device_index network interface device index
+    # @param [Hash] options
+    def network_interface(device_index, options: {})
+      options[:DeviceIndex] = device_index
+      options
+    end
+
+    # Resource
+    #
+    # @param [String] name of the resource
+    # @param [Hash] options options
+    def resource(name, options)
+      super
+      return nil unless options[:Type] && %w(AWS::EC2::Instance).include?(options[:Type])
+      output "#{name}PrivateIpAddress",
+             Description: "#{name} Private IP Address",
+             Value: get_att(name, 'PrivateIp')
+    end
+
+    # Dynamically define methods to access related parameters
+    #
+    # @param [String] name parameter key
+    # @param [Hash] options options
+    def parameter(name, options)
+      default(:Parameters, {})[name] = options
+      @parameters[name] ||= options[:Default]
+      self.class.send(:define_method, :"ref_#{name.underscore}") do
+        ref(name)
+      end
+    end
+
+    # Adds block to the run queue
+    #
+    # @param [Array] items list of blocks
+    def enqueue(items)
+      (@run_queue ||= []).concat(items || [])
+    end
+
+    # Determine content of run queue and execute each block in queue in sequence
+    def exec!
+      init_assets_dir
+      enqueue(@pre_run_blocks) if @options[:pre_run]
+      enqueue([@options[:expand] ? proc { STDOUT.puts(JSON.pretty_generate(self)) } : proc { STDOUT.puts(deploy(self)) }])
+      enqueue(@post_run_blocks) if @options[:post_run]
+      @run_queue.each(&:call) if @run_queue
+    end
+
+    # Pass generated template to underlying cloudformation client to actually create/update stack
+    # @param [TemplateDSL] template instance of template
+    # @raise [RuntimeError] when generated template exceeds 51200 size limit
+    def deploy(template)
+      template_body = template.to_json
+      if template_body.bytesize > TEMPLATE_BODY_LIMIT
+        fail("Unable to deploy template exceeding #{TEMPLATE_BODY_LIMIT} limit: #{template_body.bytesize}")
+      end
+      options = {
+        stack_name: stack_name,
+        capabilities: [@options[:capabilities]],
+        template_body: template_body
+      }
+      options[:parameters] = parameters.map { |k, v| { parameter_key: k, parameter_value: v } } unless parameters.empty?
+      action = @options[:update_stack] ? :update_stack : :create_stack
+      resp = cfn_client(region).send(action, options)
+      resp.stack_id
+    end
+  end # class RichTemplateDSL
+end # module Enscalator