elastic_beans 0.1.0

data/lib/elastic_beans/error/environments_not_ready.rb

@@ -0,0 +1,15 @@
+require "elastic_beans/error"
+
+class EnvironmentsNotReady < ElasticBeans::Error
+  def initialize(environments:)
+    @environments = environments
+  end
+
+  def message
+    msg = "Some environments are not healthy."
+    @environments.each do |environment|
+      msg << "\n    * #{environment.name} is `#{environment.status}'"
+    end
+    msg
+  end
+end

data/lib/elastic_beans/error.rb

@@ -0,0 +1,15 @@
+require "shellwords"
+
+module ElasticBeans
+  class Error < StandardError
+    protected
+
+    def command_as_string(args = nil)
+      unless args
+        args = $*.map { |arg| Shellwords.escape(arg) }.join(" ")
+      end
+
+      "beans #{args}"
+    end
+  end
+end

data/lib/elastic_beans/exec/ebextension.yml

@@ -0,0 +1,10 @@
+container_commands:
+  00_copy_files:
+    command: "mkdir -p /opt/elastic_beans && cp -vR /var/app/ondeck/.elastic_beans/exec /opt/elastic_beans/"
+  01_permissions:
+    command: "chmod 755 /opt/elastic_beans/exec/run_command.sh"
+  09_upstart:
+    command: "cp -v /opt/elastic_beans/exec/elastic_beans_exec.conf /etc/init/"
+  10_start_elastic_beans_exec:
+    command: stop elastic_beans_exec; start elastic_beans_exec
+    test: "/opt/elasticbeanstalk/bin/get-config meta -k sqsdconfig --output YAML | grep -q '^environment_name: .*-exec$'"

data/lib/elastic_beans/exec/elastic_beans_exec.conf

@@ -0,0 +1,15 @@
+description "Elastic Beans SQS consumer for running arbitrary commands"
+author "One Medical Group"
+
+respawn
+respawn limit 15 5
+
+script
+    EB_SUPPORT_DIR="`/opt/elasticbeanstalk/bin/get-config container -k support_dir`"
+    source $EB_SUPPORT_DIR/envvars
+
+    # execute using Elastic Beanstalk's built-in Ruby with aws-sdk pre-installed
+    export PATH="/opt/elasticbeanstalk/lib/ruby/bin:$PATH"
+    export GEM_PATH="/opt/elasticbeanstalk/lib/ruby/lib/ruby/2.2.0"
+    exec ruby /opt/elastic_beans/exec/init.rb start
+end script

data/lib/elastic_beans/exec/init.rb

@@ -0,0 +1,50 @@
+require "aws-sdk"
+require "fileutils"
+require "net/http"
+require "uri"
+
+class Init
+  LOGFILE = "/var/log/elastic_beans_exec.log"
+  PIDFILE = "/var/run/elastic_beans_exec.pid"
+  AZ_PATTERN = /\A(?<region>.+-\d+)\w+\z/
+  AZ_URI = URI("http://169.254.169.254/latest/meta-data/placement/availability-zone")
+
+  def self.run(command)
+    init = new
+    unless init.respond_to?(command)
+      abort "Usage: $0 {start}"
+    end
+    init.send(command)
+  end
+
+  def start
+    logging_endpoint = ENV['ELASTIC_BEANS_EXEC_LOGGING_ENDPOINT']
+    queue_url = ENV['ELASTIC_BEANS_EXEC_QUEUE_URL']
+    if queue_url.nil?
+      raise "ELASTIC_BEANS_EXEC_QUEUE_URL not set; please re-run `beans configure`."
+    end
+    sqs_client = ::Aws::SQS::Client.new(region: region)
+    require File.expand_path("../sqs_consumer", __FILE__)
+    consumer = SQSConsumer.new(
+      logfile: LOGFILE,
+      logging_endpoint: logging_endpoint,
+      queue_url: queue_url,
+      sqs: sqs_client,
+    )
+    consumer.run
+  end
+
+  private
+
+  def region
+    return @region if @region
+    az = Net::HTTP.get(AZ_URI)
+    az_match = AZ_PATTERN.match(az)
+    if az_match.nil?
+      raise "Could not find region in availability zone `#{az}'"
+    end
+    @region = az_match[:region]
+  end
+end
+
+Init.run(ARGV[0])

data/lib/elastic_beans/exec/run_command.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+# execute in the context of the application
+EB_APP_DIR="`/opt/elasticbeanstalk/bin/get-config container -k app_deploy_dir`"
+cd "$EB_APP_DIR"
+
+# load app environment
+EB_SCRIPT_DIR="`/opt/elasticbeanstalk/bin/get-config container -k script_dir`"
+EB_SUPPORT_DIR="`/opt/elasticbeanstalk/bin/get-config container -k support_dir`"
+. $EB_SUPPORT_DIR/envvars
+. $EB_SCRIPT_DIR/use-app-ruby.sh
+
+exec "$@"

data/lib/elastic_beans/exec/sqs_consumer.rb

@@ -0,0 +1,75 @@
+require "json"
+require "net/http"
+require "uri"
+
+class SQSConsumer
+  def initialize(logfile:, queue_url:, sqs:, logging_endpoint: nil)
+    @logfile = logfile
+    @queue_url = queue_url
+    @sqs = sqs
+    @logging_uri = URI(logging_endpoint) if logging_endpoint
+    @command_script = File.expand_path('../run_command.sh', __FILE__)
+  end
+
+  def run
+    trap("TERM", &method(:stop))
+    loop do
+      sleep 1
+      response = sqs.receive_message(
+        queue_url: queue_url,
+        max_number_of_messages: 1,
+      )
+      message = response.messages[0]
+      if message
+        command = JSON.parse(message.body)
+
+        @command_pid = Process.spawn("#{command_script} #{command['command']}", out: logfile, err: logfile)
+        sqs.delete_message(
+          queue_url: queue_url,
+          receipt_handle: message.receipt_handle,
+        )
+        upload_log("Executing command `#{command['command']}' on host #{`hostname`.chomp} pid #{command_pid}...")
+        _, status = Process.wait2(command_pid)
+        File.open(logfile, "a") do |file|
+          file.puts "Command `#{command['command']}' exited on host #{`hostname`.chomp}: #{status}"
+        end
+
+        @command_pid = nil
+        upload_logfile
+      end
+    end
+  end
+
+  private
+
+  attr_reader :command_pid, :command_script, :logfile, :logging_uri, :queue_url, :sqs
+
+  def stop(signal = nil)
+    signal ||= "TERM"
+    Process.kill(signal, command_pid) if command_pid
+    exit
+  rescue Errno::ESRCH
+  end
+
+  def upload_log(logs)
+    if logging_uri
+      Net::HTTP.start(logging_uri.host, logging_uri.port, use_ssl: true) do |http|
+        request = Net::HTTP::Post.new(logging_uri)
+        request.body = logs
+        request.content_type = "text/plain"
+        http.request(request)
+      end
+    end
+  end
+
+  def upload_logfile
+    if logging_uri
+      Net::HTTP.start(logging_uri.host, logging_uri.port, use_ssl: true) do |http|
+        request = Net::HTTP::Post.new(logging_uri, "Transfer-Encoding" => "chunked")
+        request.body_stream = File.open(logfile, "rb")
+        request.content_type = "text/plain"
+        http.request(request)
+      end
+    end
+  end
+    end

data/lib/elastic_beans/network.rb

@@ -0,0 +1,44 @@
+require "elastic_beans/aws/cloudformation_stack"
+
+module ElasticBeans
+  class Network
+    APPLICATION_SECURITY_GROUP_KEY = "ApplicationSecurityGroup"
+    APPLICATION_SUBNET_KEY = "ApplicationSubnet"
+    ELB_SECURITY_GROUP_KEY = "ELBSecurityGroup"
+    ELB_SUBNET_KEY = "ELBSubnet"
+    SSH_SECURITY_GROUP_KEY = "SSHSecurityGroup"
+    VPC_KEY = "VpcId"
+
+    def initialize(stack_name:, cloudformation:)
+      @stack = ElasticBeans::Aws::CloudformationStack.new(stack_name, cloudformation: cloudformation)
+    end
+
+    def application_security_groups
+      [stack.stack_output(APPLICATION_SECURITY_GROUP_KEY)]
+    end
+
+    def application_subnets
+      [stack.stack_output(APPLICATION_SUBNET_KEY)]
+    end
+
+    def elb_security_groups
+      [stack.stack_output(ELB_SECURITY_GROUP_KEY)]
+    end
+
+    def elb_subnets
+      [stack.stack_output(ELB_SUBNET_KEY)]
+    end
+
+    def ssh_security_group
+      stack.stack_output(SSH_SECURITY_GROUP_KEY)
+    end
+
+    def vpc
+      stack.stack_output(VPC_KEY)
+    end
+
+    private
+
+    attr_reader :stack
+  end
+end

data/lib/elastic_beans/rack/exec.rb

@@ -0,0 +1,63 @@
+require "logger"
+require "uri"
+require "action_dispatch"
+
+module ElasticBeans
+  module Rack
+    class Exec
+      FAILURE = ['500', {'Content-type' => 'text/plain'}, [""]]
+      SUCCESS = ['200', {'Content-type' => 'text/plain'}, [""]]
+      USER_AGENT_PREFIX = 'aws-sqsd'.freeze
+
+      def initialize(app, args={})
+        @app = app
+        @application = args.fetch(:application)
+        @sqs = args.fetch(:sqs)
+        @logger = args[:logger] || Logger.new("/dev/null")
+      end
+
+      def call(env)
+        request = ActionDispatch::Request.new(env)
+        if enabled? && cron_request?(request)
+          command = command_from_request(request)
+          logger.info("[elastic_beans] Executing command: `#{command}'")
+          begin
+            application.enqueue_command(command, sqs: sqs)
+          rescue => e
+            logger.info("[elastic_beans] Enqueue failed: #{e.class.name}: #{e.message}\n#{e.backtrace.join("; ")}")
+            return FAILURE
+          end
+
+          return SUCCESS
+        end
+
+        logger.debug { "[elastic_beans] skipped exec request: #{request.path}" }
+        app.call(env)
+      end
+
+      private
+
+      attr_reader :app, :application, :logger, :sqs
+
+      def aws_sqsd?(request)
+        current_user_agent = request.headers['User-Agent'.freeze]
+        (current_user_agent.present? &&
+          current_user_agent.size >= USER_AGENT_PREFIX.size &&
+          current_user_agent[0..(USER_AGENT_PREFIX.size - 1)] == USER_AGENT_PREFIX)
+      end
+
+      def command_from_request(request)
+        URI.unescape(request.path.sub(%r{\A/exec/}, ''))
+      end
+
+      def cron_request?(request)
+        request.local? && aws_sqsd?(request) && request.method == 'POST' && request.path.start_with?('/exec/')
+      end
+
+      def enabled?
+        worker_disabled = ENV['DISABLE_SQS_CONSUMER'] || 'false'
+        worker_disabled == 'false'
+      end
+    end
+  end
+end

data/lib/elastic_beans/scheduler/ebextension.yml

@@ -0,0 +1,4 @@
+container_commands:
+  remove_scheduler_file:
+    command: "rm -f /var/app/ondeck/cron.yaml"
+    test: "/opt/elasticbeanstalk/bin/get-config meta -k sqsdconfig --output YAML | grep '^environment_name: ' | grep -qv '^environment_name: .*-scheduler$'"

data/lib/elastic_beans/ui.rb

@@ -0,0 +1,31 @@
+module ElasticBeans
+  class UI
+    attr_reader :stdout, :stderr
+
+    def initialize(verbose: false, stdout: $stdout, stderr: $stderr)
+      @stdout = stdout
+      @stderr = stderr
+      @verbose = verbose
+    end
+
+    def debug
+      if verbose?
+        stdout.puts yield
+      end
+    end
+
+    def error(message)
+      stderr.puts message
+    end
+
+    def info(message)
+      stdout.puts message
+    end
+
+    private
+
+    def verbose?
+      !!@verbose
+    end
+  end
+end

data/lib/elastic_beans/version.rb

@@ -0,0 +1,3 @@
+module ElasticBeans
+  VERSION = "0.1.0"
+end

data/lib/elastic_beans.rb

@@ -0,0 +1,9 @@
+module ElasticBeans
+end
+
+require "elastic_beans/application"
+require "elastic_beans/application_version"
+require "elastic_beans/configuration_template"
+require "elastic_beans/environment"
+require "elastic_beans/network"
+require "elastic_beans/version"

metadata

@@ -0,0 +1,218 @@
+--- !ruby/object:Gem::Specification
+name: elastic_beans
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Adam Stegman
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-11-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
+- !ruby/object:Gem::Dependency
+  name: nesty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby-progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.19.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.19.0
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: a CLI that replaces the Elastic Beanstalk CLI, which looks great on the
+  surface but is missing some key pieces.
+email:
+- astegman@onemedical.com
+executables:
+- beans
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- Gemfile
+- LICENSE.md
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- circle.yml
+- elastic_beans.gemspec
+- exe/beans
+- lib/elastic_beans.rb
+- lib/elastic_beans/application.rb
+- lib/elastic_beans/application_version.rb
+- lib/elastic_beans/aws/cloudformation_stack.rb
+- lib/elastic_beans/command.rb
+- lib/elastic_beans/command/configure.rb
+- lib/elastic_beans/command/create.rb
+- lib/elastic_beans/command/deploy.rb
+- lib/elastic_beans/command/exec.rb
+- lib/elastic_beans/command/set_env.rb
+- lib/elastic_beans/command/talk.rb
+- lib/elastic_beans/command/version.rb
+- lib/elastic_beans/configuration_template.rb
+- lib/elastic_beans/configuration_template/base.rb
+- lib/elastic_beans/configuration_template/exec.rb
+- lib/elastic_beans/configuration_template/scheduler.rb
+- lib/elastic_beans/configuration_template/webserver.rb
+- lib/elastic_beans/configuration_template/worker.rb
+- lib/elastic_beans/dns_entry.rb
+- lib/elastic_beans/environment.rb
+- lib/elastic_beans/environment/exec.rb
+- lib/elastic_beans/environment/scheduler.rb
+- lib/elastic_beans/environment/webserver.rb
+- lib/elastic_beans/environment/worker.rb
+- lib/elastic_beans/error.rb
+- lib/elastic_beans/error/environments_not_ready.rb
+- lib/elastic_beans/exec/ebextension.yml
+- lib/elastic_beans/exec/elastic_beans_exec.conf
+- lib/elastic_beans/exec/init.rb
+- lib/elastic_beans/exec/run_command.sh
+- lib/elastic_beans/exec/sqs_consumer.rb
+- lib/elastic_beans/network.rb
+- lib/elastic_beans/rack/exec.rb
+- lib/elastic_beans/scheduler/ebextension.yml
+- lib/elastic_beans/ui.rb
+- lib/elastic_beans/version.rb
+homepage: https://github.com/onemedical/elastic_beans
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Elastic Beanstalk environment orchestration for a Rails app
+test_files: []