elastic_beans 0.1.0

data/lib/elastic_beans/application.rb

@@ -0,0 +1,127 @@
+require "json"
+require "aws-sdk"
+require "elastic_beans/aws/cloudformation_stack"
+require "elastic_beans/error"
+
+module ElasticBeans
+  class Application
+    attr_reader :name
+
+    def initialize(name:, cloudformation:, elastic_beanstalk:)
+      @name = name
+      @elastic_beanstalk = elastic_beanstalk
+      @stack = ElasticBeans::Aws::CloudformationStack.new(name, cloudformation: cloudformation)
+    end
+
+    def deployed_version
+      response = elastic_beanstalk.describe_environments(application_name: name)
+      live_environments = response.environments.select { |environment| environment.status !~ /Terminat/ }
+      environment = live_environments.max_by(&:date_updated)
+      if environment
+        ElasticBeans::ApplicationVersion.new(environment.version_label)
+      end
+    rescue ::Aws::ElasticBeanstalk::Errors::Throttling
+      sleep 5
+      retry
+    end
+
+    def configuration_templates
+      response = elastic_beanstalk.describe_applications(application_names: [name])
+      application = response.applications[0]
+      unless application
+        raise MissingApplicationError
+      end
+
+      templates = application.configuration_templates
+      templates.map { |template_name|
+        ElasticBeans::ConfigurationTemplate.new_from_existing(
+          template_name,
+          application: self,
+          elastic_beanstalk: elastic_beanstalk,
+        )
+      }
+    rescue ::Aws::ElasticBeanstalk::Errors::Throttling
+      sleep 5
+      retry
+    end
+
+    def environments
+      response = elastic_beanstalk.describe_environments(application_name: name)
+      live_environments = response.environments.select { |environment| environment.status !~ /Terminat/ }
+      live_environments.map { |environment|
+        ElasticBeans::Environment.new_from_existing(
+          environment,
+          application: self,
+          elastic_beanstalk: elastic_beanstalk,
+        )
+      }
+    rescue ::Aws::ElasticBeanstalk::Errors::Throttling
+      sleep 5
+      retry
+    end
+
+    def enqueue_command(command, sqs:)
+      if environments.none? { |environment| environment.is_a?(Environment::Exec) }
+        raise MissingExecEnvironmentError
+      end
+
+      sqs.send_message(
+        queue_url: exec_queue_url,
+        message_body: exec_message(command),
+      )
+    end
+
+    def exec_queue_url
+      stack.stack_output("ExecQueueUrl")
+    end
+
+    def versions
+      response = elastic_beanstalk.describe_application_versions(application_name: name)
+      response.application_versions.map { |version| ElasticBeans::ApplicationVersion.new(version.version_label) }
+    rescue ::Aws::ElasticBeanstalk::Errors::Throttling
+      sleep 5
+      retry
+    end
+
+    def worker_queue_url(queue)
+      queue_attribute = queue.to_s.downcase
+      queue_attribute[0] = queue_attribute[0].upcase
+      stack.stack_output("Worker#{queue_attribute}QueueUrl")
+    end
+
+    def worker_queues
+      stack.stack_outputs.each_with_object([]) do |(output_key, _), acc|
+        match = /\AWorker(?<queue>\w+)QueueUrl\z/.match(output_key)
+        if match
+          acc << match[:queue].downcase
+        end
+      end
+    end
+
+    private
+
+    attr_reader :elastic_beanstalk, :stack
+
+    def exec_message(command)
+      {
+        command: command,
+      }.to_json
+    end
+
+    class MissingApplicationError < ElasticBeans::Error
+      def message
+        "Application `#{@application_name}' does not exist. Please create the Elastic Beanstalk application using a CloudFormation stack."
+      end
+    end
+
+    class MissingExecEnvironmentError < ElasticBeans::Error
+      def message
+        <<-MESSAGE
+A one-off command cannot be executed because the "exec" environment does not exist. Please create it:
+
+    #{$0} create -a APPLICATION exec
+        MESSAGE
+      end
+    end
+  end
+end

data/lib/elastic_beans/application_version.rb

@@ -0,0 +1,202 @@
+require "fileutils"
+require "timeout"
+require "aws-sdk"
+require "zip"
+require "nesty"
+require "elastic_beans/error"
+
+module ElasticBeans
+  class ApplicationVersion
+    attr_reader :version_label
+
+    def initialize(version_label)
+      @version_label = version_label
+    end
+
+    class << self
+      def create(working_directory:, application:, elastic_beanstalk:, s3:)
+        version_label = fetch_version_label(working_directory)
+        if application.versions.any? { |version| version.version_label == version_label }
+          return new(version_label)
+        end
+
+        archive_path = File.join(working_directory, ".elasticbeanstalk", "#{version_label}.zip")
+        create_archive(sha: version_label, path: archive_path)
+        create_application_version(
+          path: archive_path,
+          version_label: version_label,
+          application: application,
+          elastic_beanstalk: elastic_beanstalk,
+          s3: s3,
+        )
+        new(version_label)
+      end
+
+      private
+
+      def create_application_version(path:, version_label:, application:, elastic_beanstalk:, s3:)
+        archive_filename = File.basename(path)
+        archive = File.new(path)
+        eb_bucket_name = find_eb_bucket_name(s3)
+        s3.put_object(body: archive, bucket: eb_bucket_name, key: archive_filename)
+        elastic_beanstalk.create_application_version(
+          application_name: application.name,
+          source_bundle: {s3_bucket: eb_bucket_name, s3_key: archive_filename},
+          version_label: version_label,
+          process: true,
+        )
+
+        status = "PROCESSING"
+        Timeout.timeout(600) do
+          while status == "PROCESSING"
+            sleep 5
+            response = elastic_beanstalk.describe_application_versions(
+              application_name: application.name,
+              version_labels: [version_label],
+            )
+            status = response.application_versions[0].status
+          end
+        end
+        if status != "PROCESSED"
+          raise FailedApplicationVersion.new(version_label: version_label, status: status)
+        end
+      rescue ::Aws::ElasticBeanstalk::Errors::Throttling
+        sleep 5
+        retry
+      rescue ::Aws::Errors::ServiceError => e
+        raise FailedApplicationVersion.new(version_label: version_label, cause: e)
+      rescue Timeout::Error
+        raise FailedApplicationVersion.new(version_label: version_label, status: status)
+      end
+
+      def create_archive(sha:, path:)
+        FileUtils.mkdir_p(File.dirname(path))
+        system("git archive --format=zip -o #{path} #{sha}")
+        unless $?.success?
+          raise FailedArchiveError.new(sha: sha, path: path)
+        end
+
+        Zip::File.open(path) do |zip_file|
+          inject_exec_into_zip(zip_file)
+          inject_scheduler_into_zip(zip_file)
+        end
+      end
+
+      def find_eb_bucket_name(s3)
+        bucket = s3.list_buckets.buckets.find { |bucket| bucket.name.start_with?("elasticbeanstalk-") }
+        unless bucket
+          raise MissingBucketError
+        end
+        bucket.name
+      end
+
+      def fetch_version_label(working_directory)
+        if Dir.glob(File.join(working_directory, ".git")).empty?
+          raise InvalidVersionWorkingDirectoryError.new(wd: working_directory)
+        end
+
+        Dir.chdir(working_directory) do
+          `git rev-parse HEAD`.chomp
+        end
+      end
+
+      def inject_exec_into_zip(zip_file)
+        begin
+          zip_file.mkdir(".elastic_beans")
+        rescue Errno::EEXIST
+        end
+        begin
+          zip_file.mkdir(".elastic_beans/exec")
+        rescue Errno::EEXIST
+        end
+        zip_file.add(
+          ".elastic_beans/exec/elastic_beans_exec.conf",
+          File.expand_path('../exec/elastic_beans_exec.conf', __FILE__),
+        )
+        zip_file.add(
+          ".elastic_beans/exec/init.rb",
+          File.expand_path('../exec/init.rb', __FILE__),
+        )
+        zip_file.add(
+          ".elastic_beans/exec/run_command.sh",
+          File.expand_path('../exec/run_command.sh', __FILE__),
+        )
+        zip_file.add(
+          ".elastic_beans/exec/sqs_consumer.rb",
+          File.expand_path('../exec/sqs_consumer.rb', __FILE__),
+        )
+
+        begin
+          zip_file.mkdir(".ebextensions")
+        rescue Errno::EEXIST
+        end
+        zip_file.add(
+          ".ebextensions/elastic_beans_exec.config",
+          File.expand_path('../exec/ebextension.yml', __FILE__),
+        )
+      end
+
+      def inject_scheduler_into_zip(zip_file)
+        begin
+          zip_file.mkdir(".ebextensions")
+        rescue Errno::EEXIST
+        end
+        zip_file.add(
+          ".ebextensions/elastic_beans_scheduler.config",
+          File.expand_path('../scheduler/ebextension.yml', __FILE__),
+        )
+      end
+    end
+
+    def ==(other)
+      version_label == other.version_label
+    end
+
+    private
+
+    class FailedApplicationVersion < ElasticBeans::Error
+      include Nesty::NestedError
+
+      def initialize(version_label:, status: nil, cause: nil)
+        @version_label = version_label
+        @status = status
+        @nested = cause
+      end
+
+      def message
+        msg = "Application version `#{@version_label}' failed"
+        msg << "\nVersion status is `#{@status}'" if @status
+        msg << "\nThe error from AWS was \"#{@nested.message}\"" if @nested
+        msg
+      end
+    end
+
+    class FailedArchiveError < ElasticBeans::Error
+      def initialize(sha:, path:)
+        @sha = sha
+        @path = path
+      end
+
+      def message
+        "Could not create git archive of sha `#{sha}' at path `#{path}'"
+      end
+    end
+
+    class InvalidVersionWorkingDirectoryError < ElasticBeans::Error
+      def initialize(wd:)
+        @wd = wd
+      end
+
+      def message
+        "Cannot create a version from '#{@wd}`, please change to a Rails project root directory."
+      end
+    end
+
+    class MissingBucketError < ElasticBeans::Error
+      def message
+        "Cannot find the Elastic Beanstalk S3 bucket." \
+          " Create an S3 bucket with a name starting with \"elasticbeanstalk-\"."
+      end
+    end
+  end
+end

data/lib/elastic_beans/aws/cloudformation_stack.rb

@@ -0,0 +1,66 @@
+require "aws-sdk"
+require "elastic_beans/error"
+
+module ElasticBeans
+  module Aws
+    class CloudformationStack
+      STACK_STATUSES = %w(CREATE_COMPLETE UPDATE_COMPLETE UPDATE_ROLLBACK_COMPLETE)
+
+      def initialize(stack_name, cloudformation:)
+        @stack_name = stack_name
+        @cloudformation = cloudformation
+      end
+
+      def stack_output(output_key)
+        output = stack.outputs.find { |o| o.output_key == output_key }
+        unless output
+          raise MissingOutputError.new(stack_name: stack_name, output_key: output_key)
+        end
+        output.output_value
+      end
+
+      def stack_outputs
+        stack.outputs.reduce({}) { |a, e| a.merge(e.output_key => e.output_value) }
+      end
+
+      private
+
+      attr_reader :stack_name, :cloudformation
+
+      def stack
+        s = cloudformation.describe_stacks(stack_name: stack_name).stacks.find { |stack|
+          STACK_STATUSES.include?(stack.stack_status)
+        }
+        unless s
+          raise MissingStackError.new(stack_name: stack_name)
+        end
+        s
+      rescue ::Aws::CloudFormation::Errors::ValidationError
+        raise MissingStackError.new(stack_name: stack_name)
+      end
+
+      class MissingStackError < ElasticBeans::Error
+        def initialize(stack_name:)
+          @stack_name = stack_name
+        end
+
+        def message
+          "CloudFormation stack `#{@stack_name}' not found." \
+            " Make sure the stack exists and matches the outputs required."
+        end
+      end
+
+      class MissingOutputError < ElasticBeans::Error
+        def initialize(stack_name:, output_key:)
+          @stack_name = stack_name
+          @output_key = output_key
+        end
+
+        def message
+          "Stack `#{@stack_name}' is missing output `#{@output_key}'." \
+            " Make sure the stack matches the outputs required."
+        end
+      end
+    end
+  end
+end

data/lib/elastic_beans/command/configure.rb

@@ -0,0 +1,184 @@
+require "elastic_beans/error/environments_not_ready"
+
+module ElasticBeans
+  module Command
+    class Configure
+      USAGE = "configure -n NETWORK -a APPLICATION [-k KEYPAIR] [-p PUBLIC_KEY] [-s SSL_CERTIFICATE_ID] [-i IMAGE_ID] [-t INSTANCE_TYPE] [-l EXEC_LOGGING_HTTP_ENDPOINT]"
+      DESC = "Configure the given Elastic Beanstalk application"
+      LONG_DESC = <<-LONG_DESC
+Configure the given Elastic Beanstalk application.
+Creates and updates configuration templates for each possible environment type.
+Updates running environments with the new configuration.
+
+Requires the networking CloudFormation stack and application name.
+Requires AWS credentials to be set in the environment, i.e. AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY.
+Some environment variables that Rails always requires must be set:
+
+* DATABASE_URL
+* SECRET_KEY_BASE
+      LONG_DESC
+
+      def initialize(
+        image_id:,
+        instance_type:,
+        keypair:,
+        logging_endpoint:,
+        public_key:,
+        ssl_certificate_id:,
+        application:,
+        network:,
+        elastic_beanstalk:,
+        iam:,
+        ui:
+      )
+        @image_id = image_id
+        @instance_type = instance_type
+        @keypair = keypair
+        @logging_endpoint = logging_endpoint
+        @public_key = public_key
+        @ssl_certificate_id = ssl_certificate_id
+        @application = application
+        @network = network
+        @elastic_beanstalk = elastic_beanstalk
+        @iam = iam
+        @ui = ui
+      end
+
+      def run
+        environments = application.environments
+        unready_environments = environments.select { |environment| environment.status != "Ready" }
+        if unready_environments.any?
+          raise EnvironmentsNotReady.new(environments: unready_environments)
+        end
+
+        progressbar = ProgressBar.create(title: "Configuring", total: nil, output: ui.stdout)
+        progressbar.log("Updating configuration templates in #{application.name}...")
+        progressbar.log("Updating base configuration template...")
+        base_config = ElasticBeans::ConfigurationTemplate::Base.new(
+          application: application,
+          elastic_beanstalk: elastic_beanstalk,
+        )
+        base_config.upsert(
+          network: network,
+          database_url: database_url,
+          secret_key_base: secret_key_base,
+          image_id: image_id,
+          instance_type: instance_type,
+          keypair: keypair,
+          iam: iam,
+        )
+        progressbar.increment
+        progressbar.log("Updating webserver configuration template...")
+        webserver_config = ElasticBeans::ConfigurationTemplate::Webserver.new(
+          application: application,
+          elastic_beanstalk: elastic_beanstalk,
+        )
+        webserver_config.upsert(
+          network: network,
+          database_url: database_url,
+          secret_key_base: secret_key_base,
+          image_id: image_id,
+          instance_type: instance_type,
+          keypair: keypair,
+          iam: iam,
+          public_key: public_key,
+          ssl_certificate_id: ssl_certificate_id,
+        )
+        progressbar.increment
+        progressbar.log("Updating exec configuration template...")
+        exec_config = ElasticBeans::ConfigurationTemplate::Exec.new(
+          application: application,
+          elastic_beanstalk: elastic_beanstalk,
+        )
+        exec_config.upsert(
+          network: network,
+          database_url: database_url,
+          secret_key_base: secret_key_base,
+          image_id: image_id,
+          instance_type: instance_type,
+          keypair: keypair,
+          iam: iam,
+          logging_endpoint: logging_endpoint,
+        )
+        progressbar.increment
+        progressbar.log("Updating scheduler configuration template...")
+        scheduler_config = ElasticBeans::ConfigurationTemplate::Scheduler.new(
+          application: application,
+          elastic_beanstalk: elastic_beanstalk,
+        )
+        scheduler_config.upsert(
+          network: network,
+          database_url: database_url,
+          secret_key_base: secret_key_base,
+          image_id: image_id,
+          instance_type: instance_type,
+          keypair: keypair,
+          iam: iam,
+        )
+        progressbar.increment
+        application.worker_queues.each do |queue|
+          progressbar.log("Updating worker-#{queue} configuration template...")
+          worker_config = ElasticBeans::ConfigurationTemplate::Worker.new(
+            queue: queue,
+            application: application,
+            elastic_beanstalk: elastic_beanstalk,
+          )
+          worker_config.upsert(
+            network: network,
+            database_url: database_url,
+            secret_key_base: secret_key_base,
+            image_id: image_id,
+            instance_type: instance_type,
+            keypair: keypair,
+            iam: iam,
+          )
+          progressbar.increment
+        end
+
+        threads = environments.map { |environment|
+          progressbar.log("Updating `#{environment.name}'...")
+          thread = Thread.new do
+            environment.update_configuration
+          end
+          progressbar.increment
+          thread
+        }
+
+        loop do
+          sleep 1
+          progressbar.increment
+          if threads.none?(&:alive?)
+            progressbar.total = progressbar.progress
+            break
+          end
+        end
+
+        threads.each(&:join)
+      end
+
+      private
+
+      attr_reader(
+        :application,
+        :image_id,
+        :instance_type,
+        :keypair,
+        :logging_endpoint,
+        :network,
+        :public_key,
+        :ssl_certificate_id,
+        :elastic_beanstalk,
+        :iam,
+        :ui,
+      )
+
+      def database_url
+        ENV['DATABASE_URL']
+      end
+
+      def secret_key_base
+        ENV['SECRET_KEY_BASE']
+      end
+    end
+  end
+end