tass 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a59b4519fd31ade584e3b01e4e2e9e710398b0d7
+  data.tar.gz: bcaf7a3d8ea6ceb7aaf543cee80cb9c9d8f56e8b
+SHA512:
+  metadata.gz: bb7a05c28477783135d2685371d8e8013f313b0d74907568ea90be27fc122b53c543a10e3c526f1ea1b7629393dccb9364b210d60fb6e7086d0b123d323f9135
+  data.tar.gz: f63a73d33066180446250213a0f6ccacf5a9e71ea6ce776edca5218871431361de810c4b150997dc6d01792e3c61334fb3a73bd0f18af3711664bb0087a9c9c5

data/bin/tass

@@ -0,0 +1,97 @@
+#!/usr/bin/env ruby
+
+require 'tapjoy/autoscaling_bootstrap'
+
+def confirm_config(aws_env, misc_config, opts)
+  use_vpc = true if misc_config[:vpc_subnets]
+  unless Tapjoy::AutoscalingBootstrap::Base.new.confirm_config(**aws_env,
+    **misc_config, **opts, use_vpc: use_vpc)
+
+    abort('Cannot continue if configuration is not correct.  Please fix.')
+  end
+end
+
+def supported_envs(listing)
+  envs = []
+  Dir.entries("#{listing}/config/common").each do |file|
+    next unless file.end_with?('yaml')
+    next if file.start_with?('defaults')
+    envs << file.chomp!('.yaml')
+  end
+  envs
+end
+
+SUB_COMMANDS = %w(create update audit)
+Trollop::options do
+  usage '[SUB_COMMAND] [options]'
+  synopsis "\nConfigures autoscaling groups.\nAvailable subcommands are: #{SUB_COMMANDS}"
+  version "#{File.basename($0)} #{Tapjoy::AutoscalingBootstrap::VERSION} \u00A9 2015 Tapjoy, Inc."
+  stop_on SUB_COMMANDS
+end
+
+cmd = ARGV.shift
+case cmd
+when 'create'
+  opts = Trollop.options do
+    # Set help message
+    usage 'create [options]'
+    synopsis 'This command creates new autoscaling groups, and overwrites existing ones.'
+
+    opt :filename, 'Specify config file to load', type: :string, required: true
+    opt :config_dir, 'Specify the directory for configuration files', type: :string, short: :none, default: Tapjoy::AutoscalingBootstrap.config_dir
+    opt :env, 'Specify which environment config to load', required: true, type: :string
+    opt :clobber_elb, 'Force ELB creation', default: false, short: :none
+    opt :clobber_as, 'Force AS group creation', default: false, short: :none
+    opt :prompt, 'Enable/disable prompts', default: true
+  end
+
+  env_list = supported_envs(opts[:config_dir])
+  unless env_list.include?(opts[:env])
+    Trollop.die :env, "Currently supported enviroments are #{env_list.join(',')}"
+  end
+
+  config, aws_env, user_data = Tapjoy::AutoscalingBootstrap::Base.new.configure_environment(opts[:filename], opts[:env], opts[:config_dir])
+  Aws.config[:region] = config[:aws_region]
+  confirm_config(aws_env, config, opts)
+
+  Tapjoy::AutoscalingBootstrap::Base.new.check_clobber(opts, config)
+  Tapjoy::AutoscalingBootstrap::AutoscalingGroup.new.create(opts, config, aws_env, user_data)
+when 'update'
+  opts = Trollop.options do
+    # Set help message
+    usage 'update [options]'
+    synopsis 'This command creates new launch configurations based on existing autoscaling groups using local instance configuration files as overrides.'
+    opt :filename, 'Specify config file to load', type: :string, required: true
+    opt :config_dir, 'Specify the directory for configuration files', type: :string, short: :none, default: Tapjoy::AutoscalingBootstrap.config_dir
+    opt :env, 'Specify which environment config to load', required: true, type: :string
+    opt :prompt, 'Enable/disable prompts', default: true
+  end
+
+  env_list = supported_envs(opts[:config_dir])
+  unless env_list.include?(opts[:env])
+    Trollop.die :env, "Currently supported enviroments are #{env_list.join(',')}"
+  end
+
+  config, aws_env, user_data = Tapjoy::AutoscalingBootstrap::Base.new.configure_environment(opts[:filename], opts[:env], opts[:config_dir])
+  Aws.config[:region] = config[:aws_region]
+  confirm_config(aws_env, config, opts)
+
+  Tapjoy::AutoscalingBootstrap::LaunchConfiguration.new(config, aws_env, user_data)
+when 'audit'
+  opts = Trollop.options do
+    usage 'audit'
+    synopsis 'This command compares local configuration files for a given cluster to the existing launch configuration and autoscaling group running in AWS.'
+    opt :filename, 'Specify config file to load', type: :string, required: true
+    opt :config_dir, 'Specify the directory for configuration files', type: :string, short: :none, default: Tapjoy::AutoscalingBootstrap.config_dir
+    opt :env, 'Specify which environment config to load', required: true, type: :string
+    opt :prompt, 'Enable/disable prompts', default: true
+  end
+
+  config, aws_env, user_data = Tapjoy::AutoscalingBootstrap::Base.new.configure_environment(opts[:filename], opts[:env], opts[:config_dir])
+  Aws.config[:region] = config[:aws_region]
+
+  config.merge!(aws_env.merge({user_data: Base64.encode64("#{user_data}")}))
+  Tapjoy::AutoscalingBootstrap::Audit.new(config)
+else
+  Trollop.educate
+end

data/lib/tapjoy/autoscaling_bootstrap/AWS/Autoscaling/group.rb

@@ -0,0 +1,61 @@
+module Tapjoy
+  module AutoscalingBootstrap
+    module AWS
+      module Autoscaling
+        # This module includes autoscaling group calls to AWS
+        module Group
+          class << self
+            def client
+              @client ||= Tapjoy::AutoscalingBootstrap::AWS::Autoscaling.client
+            end
+
+            def resize(min_size: 0, max_size: 0, desired_capacity:0)
+              self.client.update_auto_scaling_group(
+                auto_scaling_group_name: Tapjoy::AutoscalingBootstrap.scaler_name,
+                min_size: min_size, max_size: max_size,
+                desired_capacity: desired_capacity)
+            end
+
+            def delete(force_delete: true)
+              self.client.delete_auto_scaling_group(
+              auto_scaling_group_name: Tapjoy::AutoscalingBootstrap.scaler_name,
+              force_delete: force_delete)
+            end
+
+            def describe
+              self.client.describe_auto_scaling_groups(
+                auto_scaling_group_names: [
+                  Tapjoy::AutoscalingBootstrap.scaler_name
+                ]
+              )[0][0]
+            end
+
+            def create(zones:, health_check_type: nil, tags:,
+              vpc_subnets: nil, create_elb:, **unused_values)
+
+              group_hash = {
+                auto_scaling_group_name: Tapjoy::AutoscalingBootstrap.scaler_name,
+                availability_zones: zones,
+                launch_configuration_name: Tapjoy::AutoscalingBootstrap.config_name,
+                min_size: 0, max_size: 0, desired_capacity: 0,
+                termination_policies: ['OldestInstance'],
+                vpc_zone_identifier: vpc_subnets,
+                tags: Tapjoy::AutoscalingBootstrap::Autoscaling::Group.new.generate_tags(tags)
+              }
+
+              if create_elb
+                group_hash.merge!({
+                  load_balancer_names: [Tapjoy::AutoscalingBootstrap.elb_name],
+                  health_check_type: health_check_type,
+                  health_check_grace_period: 300,
+                  })
+              end
+
+              self.client.create_auto_scaling_group(**group_hash)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/tapjoy/autoscaling_bootstrap/AWS/Autoscaling/launch_config.rb

@@ -0,0 +1,45 @@
+module Tapjoy
+  module AutoscalingBootstrap
+    module AWS
+      module Autoscaling
+        # This module includes autoscaling launch config calls to AWS
+        module LaunchConfig
+          class << self
+            def client
+              @client ||= Tapjoy::AutoscalingBootstrap::AWS::Autoscaling.client
+            end
+
+            def delete
+              self.client.delete_launch_configuration(
+                launch_configuration_name: Tapjoy::AutoscalingBootstrap.config_name)
+            end
+
+            def create(image_id:, instance_type:, security_groups:, user_data:,
+              keypair:, iam_instance_profile:, classic_link_vpc_id: nil,
+              classic_link_sg_ids: nil, **unused_values)
+
+              self.client.create_launch_configuration(
+                launch_configuration_name: Tapjoy::AutoscalingBootstrap.config_name,
+                image_id: image_id,
+                iam_instance_profile: iam_instance_profile,
+                instance_type: instance_type,
+                security_groups: security_groups,
+                user_data: "#{Tapjoy::AutoscalingBootstrap::Autoscaling::Group.new.encode_user_data(user_data)}",
+                key_name: keypair,
+                classic_link_vpc_id: classic_link_vpc_id,
+                classic_link_vpc_security_groups: classic_link_sg_ids,
+              )
+            end
+
+            def describe(config_name)
+              # config_name is scoped locally, since we can't always be sure
+              # that we are using the default launch_configuration name
+              self.client.describe_launch_configurations(
+                launch_configuration_names:[config_name])[0][0]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/tapjoy/autoscaling_bootstrap/AWS/autoscaling.rb

@@ -0,0 +1,45 @@
+module Tapjoy
+  module AutoscalingBootstrap
+    module AWS
+    # This class contains AWS methods for ELB
+      module Autoscaling
+        class << self
+          def client
+            @client ||= Aws::AutoScaling::Client.new
+          end
+
+          def put_notification_configuration(sns_base_arn:, **unused_values)
+            self.client.put_notification_configuration(
+              auto_scaling_group_name: Tapjoy::AutoscalingBootstrap.scaler_name,
+              topic_arn: "#{sns_base_arn}:InstanceTerminated",
+              notification_types: ['autoscaling:EC2_INSTANCE_TERMINATE']
+            )
+          end
+
+          def put_scaling_policy(policy_name: policy, scaling_adjustment:,
+            cooldown:, **unused_values)
+
+            self.client.put_scaling_policy(policy_name: policy_name,
+              auto_scaling_group_name: Tapjoy::AutoscalingBootstrap.scaler_name,
+              scaling_adjustment: scaling_adjustment,
+              cooldown: cooldown,
+              adjustment_type: 'ChangeInCapacity'
+            )[0]
+          end
+
+          def describe_policies(policy:)
+            self.client.describe_policies(
+              auto_scaling_group_name: Tapjoy::AutoscalingBootstrap.scaler_name,
+              policy_names: [policy])
+          end
+
+          def delete_policy(policy:)
+            self.client.delete_policy(
+              auto_scaling_group_name: Tapjoy::AutoscalingBootstrap.scaler_name,
+              policy_name: policy)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/tapjoy/autoscaling_bootstrap/AWS/cloudwatch.rb

@@ -0,0 +1,41 @@
+module Tapjoy
+  module AutoscalingBootstrap
+    module AWS
+    # This class contains AWS methods for ELB
+      module Cloudwatch
+        class << self
+          def client
+            @client ||= Aws::CloudWatch::Client.new
+          end
+
+          def put_metric_alarm(alarm:, comparison_operator:,
+            evaluation_periods:, threshold:, actions:)
+            self.client.put_metric_alarm(alarm_name: alarm,
+              comparison_operator: comparison_operator,
+              evaluation_periods: evaluation_periods,
+              metric_name: 'CPUUtilization',
+              namespace: 'AWS/EC2',
+              period: 300,
+              statistic: 'Average',
+              threshold: threshold,
+              alarm_actions: actions,
+              dimensions: [
+                {
+                  name:'AutoScalingGroupName',
+                  value: Tapjoy::AutoscalingBootstrap.scaler_name
+                }
+              ])
+          end
+
+          def describe_alarm(alarm)
+            self.client.describe_alarms(alarm_names: [alarm])[0]
+          end
+
+          def delete_alarm(alarm)
+            self.client.delete_alarms(alarm_names: [alarm])
+          end
+        end
+      end
+    end
+  end
+end

data/lib/tapjoy/autoscaling_bootstrap/AWS/ec2.rb

@@ -0,0 +1,23 @@
+module Tapjoy
+  module AutoscalingBootstrap
+    module AWS
+    # This class contains AWS methods for ELB
+      module EC2
+        class << self
+          def client
+            @client ||= Aws::EC2::Client.new
+          end
+
+          def describe_security_groups(group)
+            self.client.describe_security_groups(group_names: [group])
+          end
+
+          def create_security_group(group)
+            self.client.create_security_group(group_name: group,
+              description: "Security group for #{Tapjoy::AutoscalingBootstrap.scaler_name}")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/tapjoy/autoscaling_bootstrap/AWS/elb.rb

@@ -0,0 +1,55 @@
+module Tapjoy
+  module AutoscalingBootstrap
+    module AWS
+    # This class contains AWS methods for ELB
+      module ELB
+        class << self
+          def client
+            @client ||= Aws::ElasticLoadBalancing::Client.new
+          end
+
+          # Creates ELB
+          def create(elb_protocol:, elb_port:, instance_protocol:,
+            instance_port:, zones:, **unused_values)
+            self.client.create_load_balancer(
+              load_balancer_name: Tapjoy::AutoscalingBootstrap.elb_name,
+              listeners: [
+                { protocol: elb_protocol, load_balancer_port: elb_port,
+                  instance_protocol: instance_protocol,
+                  instance_port: instance_port
+                }
+              ],
+              availability_zones: zones)
+          end
+
+          # Configures health check in AWS
+          def health_check(elb_health_target:, elb_health_interval:,
+            elb_health_timeout:, elb_unhealthy_threshold:,
+            elb_healthy_threshold:, **unused_values)
+
+            self.client.configure_health_check(
+              load_balancer_name: Tapjoy::AutoscalingBootstrap.elb_name,
+              health_check: {
+                target: elb_health_target,
+                interval: elb_health_interval,
+                timeout: elb_health_timeout,
+                unhealthy_threshold: elb_unhealthy_threshold,
+                healthy_threshold: elb_healthy_threshold
+              })
+          end
+
+          # Deletes existing ELB
+          def delete
+            self.client.delete_load_balancer(
+              load_balancer_name: Tapjoy::AutoscalingBootstrap.elb_name)
+          end
+
+          def describe
+            self.client.describe_load_balancers(
+              load_balancer_names: [Tapjoy::AutoscalingBootstrap.elb_name])
+          end
+        end
+      end
+    end
+  end
+end

data/lib/tapjoy/autoscaling_bootstrap/alerts/monitoring.rb

@@ -0,0 +1,38 @@
+module Tapjoy
+  module AutoscalingBootstrap
+    module Alerts
+      # Class to handle monitoring alerts
+      class Monitoring
+        # Create monitoring alerts
+        def create(alarm_high_alert:, alarm_low_alert:, notification:)
+
+          high_alert = {
+            alarm:               alarm_high_alert,
+            comparison_operator: 'GreaterThanOrEqualToThreshold',
+            evaluation_periods:  1,
+            threshold:           90,
+            actions:             [notification]
+          }
+
+          low_alert = {
+            alarm:               alarm_low_alert,
+            comparison_operator: 'LessThanOrEqualToThreshold',
+            evaluation_periods:  1,
+            threshold:           35,
+            actions:             [notification]
+          }
+          puts 'Clearing out original CloudWatch alarms'
+          Tapjoy::AutoscalingBootstrap::Cloudwatch.delete_alarm(alarm_high_alert)
+          Tapjoy::AutoscalingBootstrap::Cloudwatch.delete_alarm(alarm_low_alert)
+
+          puts 'Creating new CloudWatch Alarms'
+          Tapjoy::AutoscalingBootstrap::Cloudwatch.create_alarm(high_alert)
+          puts "\n"
+
+          Tapjoy::AutoscalingBootstrap::Cloudwatch.create_alarm(low_alert)
+          puts "\n"
+        end
+      end
+    end
+  end
+end

data/lib/tapjoy/autoscaling_bootstrap/alerts/scaling.rb

@@ -0,0 +1,75 @@
+module Tapjoy
+  module AutoscalingBootstrap
+    module Alerts
+      # Class to handle scaling alerts
+      class Scaling
+        # Initialize the class
+        def initialize(config)
+          @policy = Tapjoy::AutoscalingBootstrap.policy
+
+          scale_up, scale_down = prepare(**config)
+
+          create(config[:policy_up], scale_up)
+          create(config[:policy_down], scale_down)
+        end
+
+        # Prepare scaling alerts
+        def prepare(alarm_high_scale:, alarm_low_scale:, policy_up:,
+          policy_down:, scale_up_scaling_adjustment:, scale_up_cooldown:,
+          scale_up_threshold:, scale_down_scaling_adjustment:,
+          scale_down_cooldown:, scale_down_threshold:, **unused_values)
+
+          puts 'Creating autoscale alerts and policies'
+
+          puts 'Clearing out original CloudWatch alarms'
+          Tapjoy::AutoscalingBootstrap::Cloudwatch.delete_alarm(alarm_high_scale)
+          Tapjoy::AutoscalingBootstrap::Cloudwatch.delete_alarm(alarm_low_scale)
+          puts "\n"
+
+          puts 'Clearing out original scaling policies'
+          @policy.delete(policy_up)
+          @policy.delete(policy_down)
+          puts "\n"
+
+          puts 'Configuring autoscaling...'
+          scale_up = {
+            scaling_adjustment:          scale_up_scaling_adjustment,
+            cooldown:                    scale_up_cooldown,
+            threshold:                   scale_up_threshold,
+            alarm:                       alarm_high_scale,
+            comparison_operator:         'GreaterThanOrEqualToThreshold'
+          }
+
+          scale_down = {
+            scaling_adjustment:          scale_down_scaling_adjustment,
+            cooldown:                    scale_down_cooldown,
+            threshold:                   scale_down_threshold,
+            alarm:                       alarm_low_scale,
+            comparison_operator:         'LessThanOrEqualToThreshold'
+          }
+
+          return scale_up, scale_down
+        end
+
+        # Create alerts for autoscaling
+        def create(policy, scale)
+
+          puts "Creating scale policy: #{policy}"
+          scale_policy = @policy.create(policy,
+          **scale)
+
+          scale_alert = {
+            alarm:               scale[:alarm],
+            comparison_operator: scale[:comparison_operator],
+            evaluation_periods:  3,
+            threshold:           scale[:threshold],
+            actions:             [scale_policy]
+          }
+
+          puts "Creating alarms for scale policy: #{scale_policy}"
+          Tapjoy::AutoscalingBootstrap::Cloudwatch.create_alarm(scale_alert)
+        end
+      end
+    end
+  end
+end

data/lib/tapjoy/autoscaling_bootstrap/alerts.rb

@@ -0,0 +1,7 @@
+module Tapjoy
+  module AutoscalingBootstrap
+    # Module to handle alerts
+    module Alerts
+    end
+  end
+end

data/lib/tapjoy/autoscaling_bootstrap/audit.rb

@@ -0,0 +1,49 @@
+module Tapjoy
+  module AutoscalingBootstrap
+    # This class handles the comparison of local configs against upstream
+    class Audit
+      def initialize(config)
+        local_config = Hash(**clean_local_hash(config))
+
+        remote_launch_config = Tapjoy::AutoscalingBootstrap::AWS::Autoscaling::LaunchConfig.describe(Tapjoy::AutoscalingBootstrap.config_name).to_h
+        remote_group_config = Tapjoy::AutoscalingBootstrap::AWS::Autoscaling::Group.describe.to_h
+        # Combine launch config and group config into a single hash
+        remote_config = remote_launch_config.merge!(remote_group_config)
+        clean_remote_hash(remote_config)
+        puts "\n\n\n"
+        HashDiff.diff(local_config, remote_config).each do |op, key, value|
+          puts "%s %s %-60s" % [op, key, value]
+        end
+      end
+
+      private
+      # fix key names to match AWS standards
+      def clean_local_hash(config)
+        config[:key_name] = config.delete :keypair
+        config[:launch_configuration_name] = Tapjoy::AutoscalingBootstrap.config_name
+        keys = %w(bootstrap_script chef_server_url clobber clobber_elb
+          create_as_group config_dir)
+        delete_keys(config, keys)
+
+        config
+      end
+
+      # Remove keys that the local config does not support
+      def clean_remote_hash(config)
+        keys = %w(kernel_id instances desired_capacity min_size max_size
+          ramdisk_id)
+        delete_keys(config, keys)
+        config[:tags].each do |tag|
+          delete_keys(tag, %w(resource_id))
+          tag[tag[:key].to_sym] = tag.delete :value
+          delete_keys(tag, %w(resource_type propagate_at_launch key))
+        end
+      end
+
+      # Helper method to delete keys
+      def delete_keys(config, keys)
+        keys.each {|key| config.delete key.to_sym}
+      end
+    end
+  end
+end

data/lib/tapjoy/autoscaling_bootstrap/autoscaling/config.rb

@@ -0,0 +1,38 @@
+module Tapjoy
+  module AutoscalingBootstrap
+    module Autoscaling
+      # Code specific to Launch Configs
+      class Config
+
+        # Create launch configuration
+        def create(config, aws_env, user_data)
+
+          if exists
+            delete
+          else
+            puts "Launch config #{Tapjoy::AutoscalingBootstrap.config_name} does not exist, continuing..."
+          end
+
+          puts "Creating launch config: #{Tapjoy::AutoscalingBootstrap.config_name}"
+          begin
+            Tapjoy::AutoscalingBootstrap::AWS::Autoscaling::LaunchConfig.create(
+              **config, **aws_env, user_data: user_data)
+          rescue Aws::AutoScaling::Errors::ValidationError => err
+            STDERR.puts "Cannot create launch configuration: #{err}"
+          end
+        end
+
+        # Check if launch configuration exists
+        def exists
+          !Tapjoy::AutoscalingBootstrap::AWS::Autoscaling::LaunchConfig.describe(
+            Tapjoy::AutoscalingBootstrap.config_name).nil?
+        end
+
+        def delete
+          puts "Deleting launch config #{Tapjoy::AutoscalingBootstrap.config_name}"
+          Tapjoy::AutoscalingBootstrap::AWS::Autoscaling::LaunchConfig.delete
+        end
+      end
+    end
+  end
+end