tass 0.1.0

data/lib/tapjoy/autoscaling_bootstrap/autoscaling/group.rb

@@ -0,0 +1,113 @@
+module Tapjoy
+  module AutoscalingBootstrap
+    module Autoscaling
+      # Class for Autoscaling groups
+      class Group
+
+        # Initialize the class
+        def initialize
+          @scaler_name = Tapjoy::AutoscalingBootstrap.scaler_name
+          @config_name = Tapjoy::AutoscalingBootstrap.config_name
+          @elb_name    = Tapjoy::AutoscalingBootstrap.elb_name
+          @create_elb  = Tapjoy::AutoscalingBootstrap.create_elb
+        end
+
+        # Create autoscaling group
+        def create(config:, aws_env:, user_data:)
+          if exists
+            begin
+              zero_autoscale_group
+            rescue Aws::AutoScaling::Errors::ValidationError => err
+              abort("Cannot remove existing AS group #{@scaler_name}. Error: #{err}")
+            end
+          else
+            "Scaling group #{@scaler_name} does not exist, continuing..."
+          end
+
+          Tapjoy::AutoscalingBootstrap.config.create(config, aws_env,
+            user_data)
+
+          puts "Creating scaling group: #{@scaler_name}"
+          Tapjoy::AutoscalingBootstrap::AWS::Autoscaling::Group.create(**config)
+          create_termination_notification(config)
+        end
+
+        # Check if autoscale group exists
+        def exists
+          !Tapjoy::AutoscalingBootstrap::AWS::Autoscaling::Group.describe.nil?
+        end
+
+        # Encode user data into required base64 form
+        def encode_user_data(user_data)
+          Base64.encode64("#{user_data}")
+        end
+
+        # Create tags array to pass to autoscaling group
+        def generate_tags(tags)
+          tag_array = Array.new
+          return [] if tags.nil? || tags.empty?
+          tags.each do |t|
+            return [] if t.nil?
+            t.each_pair do |key, value|
+              tag_array << {
+                resource_id: @scaler_name,
+                resource_type: 'auto-scaling-group',
+                key: key.to_s,
+                value: value,
+                propagate_at_launch: true,
+              }
+            end
+          end
+          tag_array
+        end
+
+        private
+
+        # Clear out existing autoscaling group
+        def zero_autoscale_group
+          Tapjoy::AutoscalingBootstrap::AWS::Autoscaling::Group.resize
+
+          wait_for_asg_to_quiet
+
+          Tapjoy::AutoscalingBootstrap::AWS::Autoscaling::Group.delete
+
+          wait_for_asg_to_delete
+
+          abort("#{@scaler_name} still exists") if exists
+        end
+
+        # Create termination notification for Chef
+        def create_termination_notification(misc_config)
+          puts 'Create termination notification'
+          begin
+            Tapjoy::AutoscalingBootstrap::AWS::Autoscaling.put_notification_configuration(
+              **misc_config)
+          rescue Aws::AutoScaling::Errors::ValidationError => err
+            STDERR.puts "Cannot create notification: #{err}"
+          end
+        end
+
+        # Wait for ASG activities to settle before proceeding
+        def wait_for_asg_to_quiet
+          5.times do |tries|
+            puts "Waiting for scaling activities to settle (#{tries + 1} of 5)..."
+            as_group_size=Tapjoy::AutoscalingBootstrap::AWS::Autoscaling::Group.describe[:instances].length
+
+            break if as_group_size.eql?(0)
+
+            Tapjoy::AutoscalingBootstrap::Base.new.aws_wait(tries)
+          end
+        end
+
+        # Wait for ASG to be deleted
+        def wait_for_asg_to_delete
+          5.times do |tries|
+            puts "Waiting for auto scaling group to remove itself (#{tries + 1} of 5)..."
+            break unless exists
+            Tapjoy::AutoscalingBootstrap::Base.new.aws_wait(tries)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/tapjoy/autoscaling_bootstrap/autoscaling/policy.rb

@@ -0,0 +1,37 @@
+module Tapjoy
+  module AutoscalingBootstrap
+    module Autoscaling
+      # Class for Autoscaling policies
+      class Policy
+
+        # Initialize the class
+        def initialize
+          @scaler_name = Tapjoy::AutoscalingBootstrap.scaler_name
+          @group       = Tapjoy::AutoscalingBootstrap.group
+        end
+
+        # Create autoscaling policy
+        def create(policy, scale)
+          return unless Tapjoy::AutoscalingBootstrap.group.exists
+          Tapjoy::AutoscalingBootstrap::AWS::Autoscaling.put_scaling_policy(
+            policy_name: policy, **scale)
+        end
+
+        # Delete scaling policies
+        def delete(policy)
+          return unless @group.exists
+
+          if Tapjoy::AutoscalingBootstrap::AWS::Autoscaling.describe_policies(
+            policy: policy)[0].length > 0
+
+            puts "Deleting policy: #{policy}"
+            Tapjoy::AutoscalingBootstrap::AWS::Autoscaling.delete_policy(
+            policy: policy)
+          else
+            STDERR.puts "'#{policy}' doesn't exist. Skipping..."
+          end
+        end
+      end
+    end
+  end
+end

data/lib/tapjoy/autoscaling_bootstrap/autoscaling.rb

@@ -0,0 +1,6 @@
+module Tapjoy
+  module AutoscalingBootstrap
+    module Autoscaling
+    end
+  end
+end

data/lib/tapjoy/autoscaling_bootstrap/autoscaling_group.rb

@@ -0,0 +1,59 @@
+module Tapjoy
+  module AutoscalingBootstrap
+    # This class is the central launching point for new autoscaling group creation
+    class AutoscalingGroup
+      # Initialize the class
+      def create(opts, new_config, aws_env, user_data,
+
+        # ELB Parameters
+        elb_health_target: "#{new_config[:instance_protocol]}:#{new_config[:instance_port]}/healthz",
+        elb_protocol: new_config[:instance_protocol],
+        elb_name: "#{new_config[:name].gsub('_','-')}-discovery"
+      )
+
+        if new_config[:create_elb]
+          new_config.merge!({
+            elb_health_target: elb_health_target,
+            elb_protocol:      elb_protocol,
+            elb_name:          elb_name
+          })
+
+          Tapjoy::AutoscalingBootstrap::ELB.new.create(new_config, aws_env)
+        else
+          puts "\nNo ELB required"
+        end
+
+        Tapjoy::AutoscalingBootstrap::ConfigureAutoscalers.new(**new_config,
+          aws_env: aws_env, user_data: user_data, misc_config: new_config)
+
+        scaler_name = Tapjoy::AutoscalingBootstrap.scaler_name
+        config_name = Tapjoy::AutoscalingBootstrap.config_name
+
+        if new_config[:autoscale]
+          new_config.merge!({
+            alarm_low_scale:  "#{new_config[:human_name]} - Scale Down - Low CPU Utilization",
+            alarm_high_scale: "#{new_config[:human_name]} - Scale Up - High CPU Utilization",
+            policy_up:        "#{new_config[:name]}-up-policy",
+            policy_down:      "#{new_config[:name]}-down-policy"
+          })
+          Tapjoy::AutoscalingBootstrap::Alerts::Scaling.new(new_config)
+        end
+
+        if new_config[:alerts]
+          monitoring_alerts = Tapjoy::AutoscalingBootstrap::Alerts::Monitoring.new
+          monitoring_alert_config = {
+            alarm_low_alert:  "#{new_config[:human_name]} - ALARM - Extremely Low CPU Utilization",
+            alarm_high_alert: "#{new_config[:human_name]} - ALARM - Extremely High CPU Utilization",
+            notification: "#{new_config[:sns_base_arn]}:General-NOC-Notifications"
+          }
+          monitoring_alerts.create(**monitoring_alert_config)
+        end
+
+        puts "\n\nCreated -- "
+        puts "Autoscaling Group: #{scaler_name}"
+        puts "Launch Config:     #{config_name}"
+        puts 'Instance Count:    0'
+      end
+    end
+  end
+end

data/lib/tapjoy/autoscaling_bootstrap/aws.rb

@@ -0,0 +1,7 @@
+module Tapjoy
+  module AutoscalingBootstrap
+    # Module containing all references to objects in AWS SDK
+    module AWS
+    end
+  end
+end

data/lib/tapjoy/autoscaling_bootstrap/cloudwatch.rb

@@ -0,0 +1,29 @@
+module Tapjoy
+  module AutoscalingBootstrap
+    # This module configures cloudwatch alarms
+    module Cloudwatch
+      class << self
+        # Create autoscaling alarms
+        def create_alarm(scale_alert)
+          puts "Creating: #{scale_alert[:alarm]}"
+          Tapjoy::AutoscalingBootstrap::AWS::Cloudwatch.put_metric_alarm(
+            **scale_alert)
+        end
+
+        # Delete autoscaling alarms
+        def delete_alarm(alarm)
+          if self.alarm_exists(alarm)
+            puts "Deleting alarm: #{alarm}"
+            Tapjoy::AutoscalingBootstrap::AWS::Cloudwatch.delete_alarm(alarm)
+          else
+            STDERR.puts "'#{alarm}' doesn't exist. Skipping..."
+          end
+        end
+
+        def alarm_exists(alarm)
+          Tapjoy::AutoscalingBootstrap::AWS::Cloudwatch.describe_alarm(alarm).length > 0
+        end
+      end
+    end
+  end
+end

data/lib/tapjoy/autoscaling_bootstrap/configure_autoscaler.rb

@@ -0,0 +1,40 @@
+module Tapjoy
+  module AutoscalingBootstrap
+    # This class configures autoscaling groups
+    class ConfigureAutoscalers
+      # required arguments first, then optional
+      def initialize(misc_config:, aws_env:, user_data:, **unused_values)
+
+        if misc_config[:create_as_group]
+          sec_group_exists(aws_env[:security_groups]) unless misc_config[:vpc_subnets]
+          create_autoscaling_group(misc_config, aws_env, user_data)
+        else
+          puts 'Skipping creating autoscale group and launch config'
+          puts "\n"
+        end
+      end
+
+      private
+
+      # Check if security group exists and create it if it does not
+      def sec_group_exists(groups)
+        groups.each do |group|
+          begin
+            puts "Verifying #{group} exists..."
+            group = Tapjoy::AutoscalingBootstrap::AWS::EC2.describe_security_groups(group)
+          rescue Aws::EC2::Errors::InvalidGroupNotFound => err
+            STDERR.puts "Warning: #{err}"
+            puts "Creating #{group} for #{Tapjoy::AutoscalingBootstrap.scaler_name}"
+            Tapjoy::AutoscalingBootstrap::AWS::EC2.create_security_group(group)
+          end
+        end
+      end
+
+      # Create ASG logic
+      def create_autoscaling_group(misc_config, aws_env, user_data)
+        Tapjoy::AutoscalingBootstrap.group.create(config: misc_config,
+        aws_env: aws_env, user_data: user_data)
+      end
+    end
+  end
+end

data/lib/tapjoy/autoscaling_bootstrap/elb.rb

@@ -0,0 +1,56 @@
+module Tapjoy
+  module AutoscalingBootstrap
+    # This class configures elastic load balancers
+    class ELB
+      # Initialize the class
+      def initialize
+      end
+
+      # Create load balancer
+      def create(config, aws_env)
+
+        check_valid_config(config)
+        delete if exists
+        Tapjoy::AutoscalingBootstrap::AWS::ELB.create(**config, **aws_env)
+        health_check(config)
+      end
+
+      # Configure health check
+      def health_check(config)
+        abort('Target must be specified') if config[:elb_health_target].nil?
+
+        begin
+          Tapjoy::AutoscalingBootstrap::AWS::ELB.health_check(**config)
+        rescue Aws::ElasticLoadBalancing::Errors::ValidationError => err
+          abort("Fatal! Invalid ELB Configuration: #{err}")
+        end
+      end
+
+      # Check if ELB exists
+      def exists
+        begin
+          Tapjoy::AutoscalingBootstrap::AWS::ELB.describe
+          return true
+        rescue Aws::ElasticLoadBalancing::Errors::LoadBalancerNotFound => err
+          STDERR.puts "Warning: #{err}"
+          return false
+        end
+      end
+
+      # Delete ELB
+      def delete
+        puts 'Removing existing ELB'
+        Tapjoy::AutoscalingBootstrap::AWS::ELB.delete
+      end
+
+      private
+      # Check configuration
+      def check_valid_config(config)
+        fail Tapjoy::AutoscalingBootstrap::Errors::ELB::NameTooLong if Tapjoy::AutoscalingBootstrap.elb_name.length > 32
+        fail Tapjoy::AutoscalingBootstrap::Errors::ELB::NotAnELB if Tapjoy::AutoscalingBootstrap.elb_name.eql?'NaE'
+        fail Tapjoy::AutoscalingBootstrap::Errors::ELB::MissingPort if config[:elb_port].nil?
+        fail Tapjoy::AutoscalingBootstrap::Errors::ELB::MissingInstanceProtocol if config[:instance_protocol].nil?
+      end
+    end
+  end
+end

data/lib/tapjoy/autoscaling_bootstrap/errors/elb.rb

@@ -0,0 +1,48 @@
+module Tapjoy
+  module AutoscalingBootstrap
+    module Errors
+      module ELB
+        # Raise if we try to overwrite an unclobbered ELB
+        class ClobberRequired < ArgumentError
+          def initialize
+            elb_name = Tapjoy::AutoscalingBootstrap.elb_name
+            error = 'CLOBBER_ELB env var was not true, CREATE_ELB setting was ' \
+            "true and ELB '#{elb_name}' exists so we are aborting."
+            abort(error)
+          end
+        end
+
+        # Raise if ELB Name is too long
+        class NameTooLong < NameError
+          def initialize
+            elb_name = Tapjoy::AutoscalingBootstrap.elb_name
+            error = "ELB Name too long: #{elb_name.length} characters. " \
+            'Must be less than 32'
+            abort(error)
+          end
+        end
+
+        # Raise if NaE
+        class NotAnELB < ArgumentError
+          def initialize
+            abort('CREATE_ELB specified without a name')
+          end
+        end
+
+        # Raise if ELB port is missing
+        class MissingPort < ArgumentError
+          def initialize
+            abort('ELB port must be specified')
+          end
+        end
+
+        # Raise if ELB instance protocol is missing
+        class MissingInstanceProtocol < ArgumentError
+          def initialize
+            abort('Instance protocol must be specified')
+          end
+        end
+      end
+    end
+  end
+end

data/lib/tapjoy/autoscaling_bootstrap/errors.rb

@@ -0,0 +1,30 @@
+module Tapjoy
+  module AutoscalingBootstrap
+    module Errors
+      # Raise if we try to overwrite an unclobbered ASG
+      class ClobberRequired < ArgumentError
+        def initialize
+          scaler_name = Tapjoy::AutoscalingBootstrap.scaler_name
+          error = 'CLOBBER env var was not true, CREATE_AS_GROUP setting ' \
+          "true and autoscale group '#{scaler_name}' exists so we are aborting."
+          abort(error)
+        end
+      end
+
+      # Raise if an invalid environment is specified
+      class InvalidEnvironment < ArgumentError
+        def initialize
+          abort('Invalid environment specified')
+        end
+      end
+
+      # Raise if an invalid launch configuration or autoscaling group
+      # is specified
+      class InvalidAutoscalingGroup < NoMethodError
+        def initialize
+          abort("ERROR: Specified autoscaling group doesn't exist")
+        end
+      end
+    end
+  end
+end

data/lib/tapjoy/autoscaling_bootstrap/launch_configuration.rb

@@ -0,0 +1,42 @@
+module Tapjoy
+  module AutoscalingBootstrap
+    # This class is the central launching point for autoscaling group update
+    class LaunchConfiguration
+      # Initialize the class
+      def initialize(new_config, aws_env, user_data)
+        Tapjoy::AutoscalingBootstrap.scaler_name = "#{new_config[:name]}-group"
+        Tapjoy::AutoscalingBootstrap.config_name = "#{new_config[:name]}-config"
+
+        updated_config = current.to_hash.merge!(new_config)
+
+        lc_name = "#{updated_config[:launch_configuration_name].split('-config')[0]}-config_#{date_stamp}"
+        update(config_name: lc_name, user_data: user_data,
+          updated_config: updated_config, aws_env: aws_env)
+      end
+
+      def current
+        begin
+          current_config_name = Tapjoy::AutoscalingBootstrap::AWS::Autoscaling::Group.describe[:launch_configuration_name]
+        rescue NoMethodError
+          raise Tapjoy::AutoscalingBootstrap::Errors::InvalidAutoscalingGroup
+        end
+        puts "Current launch config is: #{current_config_name}\n\n"
+        Tapjoy::AutoscalingBootstrap::AWS::Autoscaling::LaunchConfig.describe(current_config_name)
+      end
+
+      def update(config_name:, scaler_name: 'NaS', user_data: user_data,
+        updated_config:, aws_env:)
+
+        Tapjoy::AutoscalingBootstrap.config_name = config_name
+        Tapjoy::AutoscalingBootstrap.scaler_name = scaler_name
+
+        Tapjoy::AutoscalingBootstrap.config.create(updated_config, aws_env,
+          user_data)
+      end
+
+      def date_stamp
+        Time.now.strftime('%Y%m%d-%H%M%S')
+      end
+    end
+  end
+end

data/lib/tapjoy/autoscaling_bootstrap/version.rb

@@ -0,0 +1,11 @@
+module Tapjoy
+  module AutoscalingBootstrap
+    module Version
+      MAJOR = 0
+      MINOR = 1
+      PATCH = 0
+    end
+
+    VERSION = [Version::MAJOR, Version::MINOR, Version::PATCH].join('.')
+  end
+end

data/lib/tapjoy/autoscaling_bootstrap.rb

@@ -0,0 +1,165 @@
+require 'aws-sdk-core'
+require 'highline/import'
+require 'base64'
+require 'yaml'
+require 'trollop'
+require 'erb'
+require 'hashdiff'
+
+# relative requires
+require_relative 'autoscaling_bootstrap/elb'
+require_relative 'autoscaling_bootstrap/autoscaling'
+require_relative 'autoscaling_bootstrap/autoscaling/config'
+require_relative 'autoscaling_bootstrap/autoscaling/policy'
+require_relative 'autoscaling_bootstrap/autoscaling/group'
+require_relative 'autoscaling_bootstrap/cloudwatch'
+require_relative 'autoscaling_bootstrap/configure_autoscaler'
+require_relative 'autoscaling_bootstrap/errors'
+require_relative 'autoscaling_bootstrap/errors/elb'
+require_relative 'autoscaling_bootstrap/alerts'
+require_relative 'autoscaling_bootstrap/alerts/scaling'
+require_relative 'autoscaling_bootstrap/alerts/monitoring'
+require_relative 'autoscaling_bootstrap/aws'
+require_relative 'autoscaling_bootstrap/AWS/elb'
+require_relative 'autoscaling_bootstrap/AWS/ec2'
+require_relative 'autoscaling_bootstrap/AWS/cloudwatch'
+require_relative 'autoscaling_bootstrap/AWS/autoscaling'
+require_relative 'autoscaling_bootstrap/AWS/Autoscaling/group'
+require_relative 'autoscaling_bootstrap/AWS/Autoscaling/launch_config'
+require_relative 'autoscaling_bootstrap/autoscaling_group'
+require_relative 'autoscaling_bootstrap/launch_configuration'
+require_relative 'autoscaling_bootstrap/version'
+require_relative 'autoscaling_bootstrap/audit'
+
+module Tapjoy
+  # Module for Autoscaling Bootstrap
+  module AutoscalingBootstrap
+    # This class is meant for class and instances variables used throughout
+    # the application.
+    class << self
+      attr_accessor :scaler_name, :config_name, :create_elb
+      attr_reader :elb_name
+
+      def elb_name=(str)
+        @elb_name = str
+      end
+
+      def policy
+        @policy = Tapjoy::AutoscalingBootstrap::Autoscaling::Policy.new
+      end
+
+      def group
+        @group = Tapjoy::AutoscalingBootstrap::Autoscaling::Group.new
+      end
+
+      def config
+        @config = Tapjoy::AutoscalingBootstrap::Autoscaling::Config.new
+      end
+
+      def cloudwatch
+        @cloudwatch = Tapjoy::AutoscalingBootstrap::CloudWatch.new
+      end
+
+      def config_dir
+        @config_dir ||= ENV['TASS_CONFIG_DIR'] || "#{ENV['HOME']}/.tass"
+      end
+    end
+
+    # Base class for generic methods used throughout the gem
+    class Base
+      # Confirm that yaml is readable and then convert to hash
+      def load_yaml(filename)
+        abort("ERROR: '#{filename}' is not readable") unless File.readable?(filename)
+        Hash[YAML.load_file(filename)]
+      end
+
+      # Using variables passed in, generate user data file
+      def generate_user_data(config)
+
+        ERB.new(
+          File.new("#{config[:config_dir]}/userdata/#{config[:bootstrap_script]}").read,nil,'-'
+        ).result(binding)
+      end
+
+      # Check if we allow clobbering and need to clobber
+      def check_clobber(opts, config)
+        fail Tapjoy::AutoscalingBootstrap::Errors::ClobberRequired if check_as_clobber(**opts, **config)
+        fail Tapjoy::AutoscalingBootstrap::Errors::ELB::ClobberRequired if check_elb_clobber(**opts, **config)
+        puts "We don't need to clobber"
+      end
+
+      # Check autoscaling clobber
+      def check_as_clobber(create_as_group:, clobber_as:, **unused_values)
+        create_as_group && Tapjoy::AutoscalingBootstrap.group.exists && !clobber_as
+      end
+
+      # Check ELB clobber
+      def check_elb_clobber(create_elb:, clobber_elb:, **unused_values)
+        elb = Tapjoy::AutoscalingBootstrap::ELB.new
+        create_elb && elb.exists && !clobber_elb
+      end
+
+      # Get AWS Environment
+      def get_security_groups(config_dir, env, group)
+
+        # Check environment file
+        unless File.readable?("#{config_dir}/config/common/#{env}.yaml")
+          fail Tapjoy::AutoscalingBootstrap::Errors::InvalidEnvironment
+        end
+
+        security_groups = {security_groups: group.split(',')}
+      end
+
+      # Confirm config settings before running autoscaling code
+      def confirm_config(keypair:, zones:, security_groups:, instance_type:,
+        image_id:, iam_instance_profile:, prompt:, use_vpc: use_vpc,
+        vpc_subnets: nil, **unused_values)
+
+        elb_name = Tapjoy::AutoscalingBootstrap.elb_name
+
+        puts '  Preparing to configure the following autoscaling group:'
+        puts "  Launch Config:  #{Tapjoy::AutoscalingBootstrap.config_name}"
+        puts "  Auto Scaler:    #{Tapjoy::AutoscalingBootstrap.scaler_name}"
+        puts "  ELB:            #{elb_name}" unless elb_name.eql? 'NaE'
+        puts "  Key Pair:       #{keypair}"
+        puts "  Zones:          #{zones.join(',')}"
+        puts "  Groups:         #{security_groups.sort.join(',')}"
+        puts "  Instance Type:  #{instance_type}"
+        puts "  Image ID:       #{image_id}"
+        puts "  IAM Role:       #{iam_instance_profile}"
+        puts "  VPC Subnets:    #{vpc_subnets}" if use_vpc
+
+        puts "\n\nNOTE! Continuing may have adverse effects if you end up " \
+        "deleting an IN-USE PRODUCTION scaling group. Don't be dumb."
+        return true unless prompt
+        agree('Is this information correct? [y/n]')
+      end
+
+      # configure environment
+      def configure_environment(filename, env, config_dir)
+        defaults_hash = self.load_yaml("#{config_dir}/config/common/defaults.yaml")
+        facet_hash    = self.load_yaml("#{config_dir}/config/clusters/#{filename}")
+        env_hash      = self.load_yaml("#{config_dir}/config/common/#{env}.yaml")
+
+        new_config = defaults_hash.merge!(env_hash).merge(facet_hash)
+        new_config[:config_dir] = config_dir
+        aws_env = self.get_security_groups(config_dir, env, new_config[:group])
+
+        Tapjoy::AutoscalingBootstrap.scaler_name = "#{new_config[:name]}-group"
+        Tapjoy::AutoscalingBootstrap.config_name = "#{new_config[:name]}-config"
+        # If there's no ELB, then Not a ELB
+        puts new_config[:elb_name]
+        Tapjoy::AutoscalingBootstrap.elb_name = new_config[:elb_name] || 'NaE'
+        Tapjoy::AutoscalingBootstrap.create_elb = new_config[:create_elb]
+        user_data = self.generate_user_data(new_config)
+        return new_config, aws_env, user_data
+      end
+
+      # Exponential backup
+      def aws_wait(tries)
+        puts "Sleeping for #{2 ** tries}..."
+        sleep 2 ** tries
+      end
+    end
+  end
+end