elastic_beans 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 18b8fddb84235084c0ed2656617d432b4b22e6d4
+  data.tar.gz: c49a0e1f3240697c150aab493b9373913dc25c61
+SHA512:
+  metadata.gz: 480c7aeed8133114eafa17306f832a0a1e21487c9b0069424567283122b166fb3051eaee5f7d7b3829f62030fc91c2904af0bd65430087603884593dda108858
+  data.tar.gz: 35cc62acbf16cdbe0b26b2a600b9f84332bb6c5ac7e151053b56072aa40c4692f37245293285a0d671287ede3791f8cdce678ef53e662f9bae43693253371e5c

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/Gemfile

@@ -0,0 +1,9 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in elastic_beans.gemspec
+gemspec
+
+group :test do
+  gem "net-ssh-gateway"
+  gem "rails", "~> 5.0"
+end

data/LICENSE.md

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 One Medical Group, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,184 @@
+# Elastic Beans
+
+Elastic Beans orchestrates an Elastic Beanstalk application for a Rails app.
+It is a CLI that replaces the [Elastic Beanstalk CLI][eb], which looks great on the surface but is missing some key pieces.
+
+* The VPC support in `eb create` is great, but it ends there. Connecting to instances inside a VPC requires manual SSH gateway setup.
+* The CLI [supports multiple environments][eb-compose], but only in a rigid fashion that isn't usable when those environments share the same codebase.
+
+[eb]: http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/eb-cli3.html
+[eb-compose]: http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/ebcli-compose.html
+
+## Usage
+
+Elastic Beans makes heavy use of the [AWS SDK][aws-sdk].
+Configuration of AWS credentials and region should be completed before using this gem, according to [the SDK configuration instructions][configuration].
+As the SDK documentation suggests, using environment variables is recommended.
+
+[aws-sdk]: https://aws.amazon.com/sdk-for-ruby/
+[configuration]: http://docs.aws.amazon.com/sdk-for-ruby/latest/DeveloperGuide/aws-ruby-sdk-getting-started.html#aws-ruby-sdk-configuration
+
+    # Pre-configure the application before creating environments
+    # Rails will not start without DATABASE_URL and SECRET_TOKEN_BASE
+    export DATABASE_URL=mysql2://... SECRET_KEY_BASE=abc123...
+    beans configure -n myapp-networking -a myapp -k KEYPAIR -s SSL_CERTIFICATE_ID [-i IMAGE_ID] [-t INSTANCE_TYPE] [-l LOGGING_HTTP_ENDPOINT]
+
+    # Create a webserver environment with a pretty DNS name at myapp.TLD (managed by Route53)
+    beans create -a myapp [-d myapp.TLD] [--tags=Environment:production Team:Unicorn] webserver
+
+    # Create a worker environment using an SQS queue created by the CloudFormation template
+    beans create -a myapp [-q QUEUE] worker
+
+    # Create an execution environment using an SQS queue created by the CloudFormation template
+    beans create -a myapp exec
+
+    # Create a periodic task scheduler using cron.yaml
+    beans create -a myapp scheduler
+
+    # Set environment variables across all environments
+    beans setenv -a myapp REDIS_URL=redis://...
+
+    # Deploy the HEAD git commit to all environments
+    # Creates an application version named after the git ref
+    beans deploy -a myapp
+
+    # Run one-off tasks
+    beans exec -a myapp rake db:migrate
+
+    # Update all existing environments and configuration
+    beans configure -n myapp-networking -a myapp [-k KEYPAIR] [-s SSL_CERTIFICATE_ID] [-i IMAGE_ID] [-t INSTANCE_TYPE]
+
+### API
+
+    app = ElasticBeans::Application.new(
+      name: "myapp",
+      cloudformation: Aws::CloudFormation::Client.new,
+      elastic_beanstalk: Aws::ElasticBeanstalk::Client.new,
+    )
+    app.exec_command("rake db:migrate", sqs: Aws::SQS::Client.new)
+
+### Periodic tasks
+
+Elastic Beanstalk [supports periodic tasks][periodic] by sending a POST request to an endpoint of your application.
+Elastic Beans includes a middleware that can convert specially-crafted paths into commands and enqueue them for execution.
+The middleware only accepts localhost requests from `aws-sqsd`.
+
+[periodic]: http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/using-features-managing-env-tiers.html#worker-periodictasks
+
+In `config/initializers/elastic_beans.rb`, add the middleware into your stack, below forcing HTTPS (`aws-sqsd` does not use HTTPS):
+
+    require "elastic_beans/rack/exec"
+
+    if Rails.configuration.force_ssl
+      Rails.configuration.middleware.insert_before(
+        ActionDispatch::SSL,
+        ElasticBeans::Rack::Exec,
+        application: ElasticBeans::Application.new(
+          name: "myapp",
+          cloudformation: Aws::CloudFormation::Client.new,
+          elastic_beanstalk: Aws::ElasticBeanstalk::Client.new,
+        ),
+        sqs: Aws::SQS::Client.new,
+        logger: Rails.logger,
+      )
+    else
+      Rails.configuration.middleware.use(
+        ElasticBeans::Rack::Exec,
+        application: ElasticBeans::Application.new(
+          name: "myapp",
+          cloudformation: Aws::CloudFormation::Client.new,
+          elastic_beanstalk: Aws::ElasticBeanstalk::Client.new,
+        ),
+        sqs: Aws::SQS::Client.new,
+        logger: Rails.logger,
+      )
+    end
+
+In `cron.yaml`, URL-encode the full command to be run and add it as another path segment to `/exec`:
+
+    - name: rake myapp:mytask[arg1,arg2]
+      url: "/exec/rake%20myapp%3Amytask%5Barg1%2Carg2%5D"
+
+Then create a periodic scheduler environment using `cron.yaml`:
+
+    beans create -a myapp scheduler
+
+This environment will enqueue the commands from `cron.yaml` for the `exec` environment to run.
+
+## Requirements
+
+Elastic Beans assumes that you use CloudFormation to manage your infrastructure.
+It requires two stacks for each application: a networking stack, and an application stack.
+The settings for your application are discovered from the [outputs of these CloudFormation stacks][outputs].
+
+[outputs]: http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/outputs-section-structure.html
+
+The networking stack can be shared between many applications, but each application should have its own application stack.
+For example, you might create separate "eb-development" and "eb-production" network stacks.
+Then, create an Elastic Beanstalk application from a stack named "myapp."
+Finally, you can set it up with beans:
+
+    $ beans configure -n eb-development -a myapp ...
+
+### Networking
+
+Elastic Beans uses VPC networking and will run your application from a private subnet.
+Networking settings are discovered from a CloudFormation stack that you must set up before running the `configure` command.
+It must have the following outputs, which will be used for the corresponding settings:
+
+* `ApplicationSecurityGroup` - [`aws:autoscaling:launchconfiguration/SecurityGroups`][launchconfiguration]
+    * Allow TCP connections from the `ELBSecurityGroup` on ports 80 and 443
+* `ApplicationSubnet` - [`aws:ec2:vpc/Subnets`][vpc]
+* `ELBSecurityGroup` - [`aws:elb:loadbalancer/SecurityGroups`][loadbalancer], [`aws:elb:loadbalancer/ManagedSecurityGroup`][loadbalancer]
+    * Allow TCP connections on ports 80 and 443 from `0.0.0.0/0`
+* `ELBSubnet` - [`aws:ec2:vpc/ELBSubnets`][vpc]
+* `SSHSecurityGroup` - [`aws:ec2:vpc/Subnets`][vpc]
+    * Allow TCP connections from your VPC on ports 80 and 443
+* `VpcId` - [`aws:ec2:vpc/VPCId`][vpc]
+
+[launchconfiguration]: http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/command-options-general.html#command-options-general-autoscalinglaunchconfiguration
+[loadbalancer]: http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/command-options-general.html#command-options-general-elbloadbalancer
+[vpc]: http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/command-options-general.html#command-options-general-ec2vpc
+
+You must create [the default instance profile and service role for Elastic Beanstalk](http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/AWSHowTo.iam.html).
+Elastic Beans will use the default names: `aws-elasticbeanstalk-ec2-role` and `aws-elasticbeanstalk-service-role`.
+
+### Application
+
+Your Elastic Beanstalk application must already exist and also be created using CloudFormation.
+Its details will be discovered from the following outputs:
+
+* `ExecQueueUrl`
+* `Worker[Name]QueueUrl`
+
+Where a separate worker environment will be configured for each queue `[Name]` that appears.
+The `Default` name must be present.
+
+### Code
+
+Your application must use the [active-elastic-job gem](https://github.com/tawan/active-elastic-job) for background job processing.
+Elastic Beans will set the `DISABLE_SQS_CONSUMER` environment variable appropriately in your environments.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'elastic_beans'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install elastic_beans
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/onemedical/elastic_beans.

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "elastic_beans"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/circle.yml

@@ -0,0 +1,20 @@
+---
+dependencies:
+  override:
+    - |
+      case $CIRCLE_NODE_INDEX in
+        0)
+          rvm-exec 2.3.1 bash -c "bundle check --path=vendor/bundle_2.3 || bundle install --path=vendor/bundle_2.3 --jobs=4 --retry=3"
+          ;;
+        1)
+          rvm-exec 2.2.5 bash -c "bundle check --path=vendor/bundle_2.2 || bundle install --path=vendor/bundle_2.2 --jobs=4 --retry=3"
+          ;;
+      esac
+  post:
+  - git config --global user.name "CircleCI"
+  - git config --global user.email "ops-ro@onemedical.com"
+
+test:
+  override:
+    - case $CIRCLE_NODE_INDEX in 0) rvm-exec 2.3.1 bash -c "bundle check --path=vendor/bundle_2.3 && bundle exec rake" ;; 1) rvm-exec 2.2.5 bash -c "bundle check --path=vendor/bundle_2.2 && bundle exec rake" ;; esac:
+        parallel: true

data/elastic_beans.gemspec

@@ -0,0 +1,31 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'elastic_beans/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "elastic_beans"
+  spec.version       = ElasticBeans::VERSION
+  spec.authors       = ["Adam Stegman"]
+  spec.email         = ["astegman@onemedical.com"]
+
+  spec.summary       = "Elastic Beanstalk environment orchestration for a Rails app"
+  spec.description   = "a CLI that replaces the Elastic Beanstalk CLI, which looks great on the surface but is missing some key pieces."
+  spec.homepage      = "https://github.com/onemedical/elastic_beans"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "aws-sdk", "~> 2.3"
+  spec.add_dependency "nesty", "~> 1.0"
+  spec.add_dependency "rails"
+  spec.add_dependency "ruby-progressbar", "~> 1.2"
+  spec.add_dependency "rubyzip", "~> 1.2"
+  spec.add_dependency "thor", "~> 0.19.0"
+
+  spec.add_development_dependency "bundler", "~> 1.12"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+end

data/exe/beans

@@ -0,0 +1,198 @@
+#!/usr/bin/env ruby
+
+require "aws-sdk"
+require "thor"
+require "elastic_beans"
+require "elastic_beans/error"
+require "elastic_beans/command"
+require "elastic_beans/ui"
+
+class ElasticBeans::CLI < Thor
+  class_option :verbose, type: :boolean, aliases: %w(-v)
+
+  desc ElasticBeans::Command::Configure::USAGE, ElasticBeans::Command::Configure::DESC
+  long_desc ElasticBeans::Command::Configure::LONG_DESC
+  option :application, aliases: %w(-a), required: true
+  option :network, aliases: %w(-n), required: true
+  option :image_id, aliases: %w(-i)
+  option :instance_type, aliases: %w(-t)
+  option :keypair, aliases: %w(-k)
+  option :logging_endpoint, aliases: %w(-l)
+  option :public_key, aliases: %w(-p)
+  option :ssl_certificate_id, aliases: %w(-s)
+  def configure
+    @verbose = options[:verbose]
+    ElasticBeans::Command::Configure.new(
+      image_id: options[:image_id],
+      instance_type: options[:instance_type],
+      keypair: options[:keypair],
+      logging_endpoint: options[:logging_endpoint],
+      public_key: options[:public_key],
+      ssl_certificate_id: options[:ssl_certificate_id],
+      application: application(name: options[:application]),
+      network: network(stack_name: options[:network]),
+      elastic_beanstalk: elastic_beanstalk_client,
+      iam: iam_client,
+      ui: ui,
+    ).run
+  rescue StandardError => e
+    error(e)
+  end
+
+  desc ElasticBeans::Command::Create::USAGE, ElasticBeans::Command::Create::DESC
+  long_desc ElasticBeans::Command::Create::LONG_DESC
+  option :application, aliases: %w(-a), required: true
+  option :dns, aliases: %w(-d)
+  option :queue, aliases: %w(-q)
+  option :tags, type: :hash, default: {}
+  def create(environment_name)
+    @verbose = options[:verbose]
+    ElasticBeans::Command::Create.new(
+      environment_name: environment_name,
+      dns: options[:dns],
+      queue: options[:queue],
+      tags: options[:tags],
+      application: application(name: options[:application]),
+      elastic_beanstalk: elastic_beanstalk_client,
+      route53: route53_client,
+      s3: s3_client,
+      ui: ui,
+    ).run
+  rescue StandardError => e
+    error(e)
+  end
+
+  desc ElasticBeans::Command::Deploy::USAGE, ElasticBeans::Command::Deploy::DESC
+  long_desc ElasticBeans::Command::Deploy::LONG_DESC
+  option :application, aliases: %w(-a), required: true
+  def deploy
+    @verbose = options[:verbose]
+    ElasticBeans::Command::Deploy.new(
+      application: application(
+        name: options[:application],
+      ),
+      elastic_beanstalk: elastic_beanstalk_client,
+      s3: s3_client,
+      ui: ui,
+    ).run
+  rescue StandardError => e
+    error(e)
+  end
+
+  desc ElasticBeans::Command::Exec::USAGE, ElasticBeans::Command::Exec::DESC
+  long_desc ElasticBeans::Command::Exec::LONG_DESC
+  option :application, aliases: %w(-a), required: true
+  def exec(*command_parts)
+    @verbose = options[:verbose]
+    ElasticBeans::Command::Exec.new(
+      application: application(
+        name: options[:application],
+      ),
+      sqs: sqs_client,
+      ui: ui,
+    ).run(*command_parts)
+  rescue StandardError => e
+    error(e)
+  end
+
+  desc ElasticBeans::Command::SetEnv::USAGE, ElasticBeans::Command::SetEnv::DESC
+  long_desc ElasticBeans::Command::SetEnv::LONG_DESC
+  option :application, aliases: %w(-a), required: true
+  def setenv(*env_pairs)
+    @verbose = options[:verbose]
+    ElasticBeans::Command::SetEnv.new(
+      application: application(
+        name: options[:application],
+      ),
+      ui: ui,
+    ).run(*env_pairs)
+  rescue StandardError => e
+    error(e)
+  end
+
+  desc "talk", ""
+  def talk
+    ElasticBeans::Command::Talk.new(ui: ui).run
+  end
+
+  map %w[--version] => :__print_version
+  desc "--version", "Print the version and exit"
+  def __print_version
+    ElasticBeans::Command::Version.new(ui: ui).run
+  end
+
+  private
+
+  def application(
+    name:,
+    cloudformation: cloudformation_client,
+    elastic_beanstalk: elastic_beanstalk_client
+  )
+    @application ||= ElasticBeans::Application.new(
+      name: name,
+      cloudformation: cloudformation,
+      elastic_beanstalk: elastic_beanstalk,
+    )
+  end
+
+  def cloudformation_client
+    ::Aws::CloudFormation::Client.new
+  end
+
+  def elastic_beanstalk_client
+    ::Aws::ElasticBeanstalk::Client.new
+  end
+
+  def error(e)
+    ui.debug { e.inspect }
+    ui.debug { e.backtrace }
+
+    case e
+    when ElasticBeans::Error
+      ui.error(e.message)
+    when ::Aws::Errors::MissingCredentialsError
+      ui.error("Missing AWS credentials, please set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY and try again.")
+    else
+      raise e
+    end
+
+    exit 1
+  end
+
+  def iam_client
+    ::Aws::IAM::Client.new
+  end
+
+  def network(stack_name:, cloudformation: cloudformation_client)
+    @network ||= ElasticBeans::Network.new(
+      stack_name: stack_name,
+      cloudformation: cloudformation,
+    )
+  end
+
+  def route53_client
+    ::Aws::Route53::Client.new
+  end
+
+  def s3_client
+    ::Aws::S3::Client.new
+  end
+
+  def sqs_client
+    ::Aws::SQS::Client.new
+  end
+
+  def ui
+    ElasticBeans::UI.new(verbose: verbose?)
+  end
+
+  def verbose?
+    @verbose
+  end
+
+  def self.exit_on_failure?
+    true
+  end
+end
+
+ElasticBeans::CLI.start(ARGV)