elastic_beans 0.10.0.alpha1 → 0.10.0.alpha2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 44f1c4f70c8a932819315f3d02623b75423ff299
-  data.tar.gz: 270d9d55029d816b5f45f91b13d2ff3e89e6b456
+  metadata.gz: 9c681be8f1584e92ac3326f36602a6f006739fae
+  data.tar.gz: 2d7efd88e9401d2c8ff327f15992de92ba94323d
 SHA512:
-  metadata.gz: c9808cfe16431ef5bd7ac8df11ac31e85331d36c0fefb89d24c6aa95bd91ca593762e09db378c3913a29bd0dd4d732cc4e0b831aaf39645684cbd123cc720acc
-  data.tar.gz: e223d9b76c7b922dd3271958458e86a100eaf45842c47012b33de0d4b647fc405f68d009f56daa7cb9d37365dec7230181b93631240fb0bbf0de8a26daad345c
+  metadata.gz: 03076cb44db365dcf4d20fdfe45aad0a1d845b9ad346fdb8b6074be7073045c4f94fe411e637aafd965ddce9bf54dbd03cde18f1e675d89ce75fbde5be9e462d
+  data.tar.gz: fefcc5407eb3a0f5a19e317b85fb1f6c2aa0453d5ad1513648a5260fb3e01b9b55e17489641e2a9e0f63b8da6544f15f6825c16ba07bc52eab1aa2c1e77fc8bc

data/.circleci/config.yml

@@ -0,0 +1,20 @@
+version: 2
+jobs:
+  build:
+    docker:
+    - image: ruby:latest
+    working_directory: /elastic_beans
+    steps:
+    - checkout
+    - run:
+        name: Configure git
+        command: |
+          git config --global user.name "CircleCI"
+          git config --global user.email "ops-ro@onemedical.com"
+    - run: bundle install --jobs=4 --retry=3
+    - run:
+        command: bundle exec rake
+        environment:
+          SPEC_OPTS: --format progress --format RspecJunitFormatter -o tmp/test_results/rspec/junit.xml
+    - store_test_results:
+        path: tmp/test_results/

data/elastic_beans.gemspec

@@ -25,7 +25,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "ruby-progressbar", "~> 1.2"
   spec.add_dependency "rubyzip", "~> 1.2"
   spec.add_dependency "thor", "~> 0.19.0"
-  spec.add_dependency "tty-table", "~> 0.7.0"
+  spec.add_dependency "tty-table", "~> 0.8.0"
 
   spec.add_development_dependency "bundler", "~> 1.12"
   spec.add_development_dependency "rake", "~> 10.0"

data/lib/elastic_beans/application.rb

@@ -3,6 +3,7 @@ require "timeout"
 require "aws-sdk"
 require "elastic_beans/aws/cloudformation_stack"
 require "elastic_beans/error"
+require "elastic_beans/error/access_denied"
 
 module ElasticBeans
   # An Elastic Beanstalk application which should exist.
@@ -42,7 +43,7 @@ module ElasticBeans
       response = elastic_beanstalk.describe_applications(application_names: [name])
       application = response.applications[0]
       unless application
-        raise MissingApplicationError
+        raise MissingApplicationError.new(application: self)
       end
 
       templates = application.configuration_templates
@@ -61,7 +62,7 @@ module ElasticBeans
     # Returns the ElasticBeans::EnvVars for this application.
     def env_vars
       unless exists?
-        raise MissingApplicationError
+        raise MissingApplicationError.new(application: self)
       end
 
       EnvVars.new(application: self, s3: s3)
@@ -96,6 +97,8 @@ module ElasticBeans
         raise MissingBucketError
       end
       @bucket_name = bucket.name
+    rescue ::Aws::S3::Errors::AccessDenied
+      raise AccessDeniedS3Error.new
     end
 
     # Enqueues a one-off Exec::Command to be run on the application's +exec+ environment.
@@ -112,45 +115,31 @@ module ElasticBeans
       end
       command.metadata[:bucket] = bucket_name
       command.metadata[:key] = "#{command_key_prefix}#{command.id}.json"
+      s3.put_object(
+        bucket: bucket_name,
+        key: command.metadata[:key],
+        body: command.to_json,
+      )
 
       sqs.send_message(
         queue_url: exec_queue_url,
         message_body: command.to_json,
       )
+    rescue ::Aws::S3::Errors::AccessDenied
+      raise AccessDeniedS3Error.new(bucket: bucket_name, key: command.metadata[:key])
     end
 
-    # Fetches commands enqueued in the +exec_queue_url+.
-    # Polls the queue for 5 seconds.
-    #
-    # If a large number of commands is enqueued they may not all be found.
+    # Fetches up to 100 previously-enqueued commands that are running or scheduled to run.
+    # Commands are deserialized from metadata files in a well-known location in S3.
+    # The metadata is created when the command is enqueued.
+    # The instances in the exec environment update the metadata when executing a command, and remove the metadata when they are done.
     #
-    # Raises an error if the exec queue cannot be found.
+    # Raises an error if the application does not exist.
     def enqueued_commands
-      messages = []
-      Timeout.timeout(5) do
-        loop do
-          messages += sqs.receive_message(
-            queue_url: exec_queue_url,
-            max_number_of_messages: 10,
-            visibility_timeout: 5,
-            wait_time_seconds: 5,
-          ).messages
-        end
+      unless exists?
+        raise MissingApplicationError.new(application: self)
       end
-    rescue Timeout::Error
-      messages.map { |msg| Exec::Command.from_json(msg.body) }
-    end
 
-    def exec_queue_url
-      stack.stack_output("ExecQueueUrl")
-    end
-
-    # Fetches commands running in the Environment::Exec environment.
-    # Commands are deserialized from metadata files in a well-known location in S3.
-    # The instances update the metadata when executing a command, and remove the metadata when they are done.
-    #
-    # Raises an error if the exec environment cannot be found.
-    def running_commands
       # Ignoring truncation for simplicity; >100 commands will just have to suffer.
       objects = s3.list_objects_v2(
         bucket: bucket_name,
@@ -168,6 +157,36 @@ module ElasticBeans
       }
     end
 
+    # Fetches the +ExecQueueUrl+ Output from the application CloudFormation stack.
+    # The stack must have the same name as the application.
+    def exec_queue_url
+      stack.stack_output("ExecQueueUrl")
+    end
+
+    # Schedules the given ElasticBeans::Exec::Command or +id+ for termination.
+    # Removes the metadata for the command and relies on the SQSConsumer to terminate it.
+    #
+    # Raises an error if the application does not exist.
+    def kill_command(command_or_id)
+      unless exists?
+        raise MissingApplicationError.new(application: self)
+      end
+
+      if command_or_id.is_a?(ElasticBeans::Exec::Command)
+        command_id = command_or_id.id
+      else
+        command_id = command_or_id
+      end
+
+      key = "#{command_key_prefix}#{command_id}.json"
+      s3.delete_object(
+        bucket: bucket_name,
+        key: key,
+      )
+    rescue ::Aws::S3::Errors::AccessDenied
+      raise AccessDeniedS3Error.new(bucket: bucket_name, key: key)
+    end
+
     # Returns an ElasticBeans::ApplicationVersion for each version of the Elastic Beanstalk application.
     def versions
       response = elastic_beanstalk.describe_application_versions(application_name: name)
@@ -199,6 +218,7 @@ module ElasticBeans
 
     attr_reader :elastic_beanstalk, :s3, :stack
 
+    # Returns the S3 prefix under which command metadata is stored.
     def command_key_prefix
       @command_key_prefix ||= "#{name}/exec/command/"
     end
@@ -217,8 +237,12 @@ module ElasticBeans
     # :nodoc: all
     # @!visibility private
     class MissingApplicationError < ElasticBeans::Error
+      def initialize(application:)
+        @application = application
+      end
+
       def message
-        "Application `#{@application_name}' does not exist. Please create the Elastic Beanstalk application using a CloudFormation stack."
+        "Application `#{@application.name}' does not exist. Please create the Elastic Beanstalk application using a CloudFormation stack."
       end
     end
 

data/lib/elastic_beans/aws/cloudformation_stack.rb

@@ -1,5 +1,6 @@
 require "aws-sdk"
 require "elastic_beans/error"
+require "elastic_beans/error/access_denied"
 
 module ElasticBeans
   # :nodoc: all
@@ -39,6 +40,8 @@ module ElasticBeans
         s
       rescue ::Aws::CloudFormation::Errors::ValidationError
         raise MissingStackError.new(stack_name: stack_name)
+      rescue ::Aws::CloudFormation::Errors::AccessDenied
+        raise AccessDeniedCloudFormationError.new(stack_name: stack_name)
       end
 
       class MissingStackError < ElasticBeans::Error

data/lib/elastic_beans/cli.rb

@@ -98,6 +98,21 @@ class ElasticBeans::CLI < Thor
     error(e)
   end
 
+  desc ElasticBeans::Command::Kill::USAGE, ElasticBeans::Command::Kill::DESC
+  long_desc ElasticBeans::Command::Kill::LONG_DESC
+  option :application, aliases: %w(-a), required: true, desc: APPLICATION_DESC
+  def kill(command_id)
+    @verbose = options[:verbose]
+    ElasticBeans::Command::Kill.new(
+      application: application(
+        name: options[:application],
+      ),
+      ui: ui,
+    ).run(command_id)
+  rescue StandardError => e
+    error(e)
+  end
+
   desc ElasticBeans::Command::Ps::USAGE, ElasticBeans::Command::Ps::DESC
   long_desc ElasticBeans::Command::Ps::LONG_DESC
   option :application, aliases: %w(-a), required: true, desc: APPLICATION_DESC

data/lib/elastic_beans/command/exec.rb

@@ -21,8 +21,9 @@ Requires AWS credentials to be set in the environment, i.e. AWS_ACCESS_KEY_ID an
       end
 
       def run(*command_parts)
-        ui.info("Running `#{command_parts.join(" ")}' on #{application.name}...")
-        application.enqueue_command(command(command_parts))
+        command = command(command_parts)
+        ui.info("Running `#{command.command_string}' on #{application.name}... (ID=#{command.id})")
+        application.enqueue_command(command)
       end
 
       private

data/lib/elastic_beans/command/kill.rb

@@ -0,0 +1,38 @@
+require "aws-sdk"
+require "elastic_beans/error/access_denied"
+
+module ElasticBeans
+  module Command
+    # :nodoc: all
+    class Kill
+      USAGE = "kill COMMAND_ID"
+      DESC = "Kill a running command or cancel a scheduled command that was enqueued with `exec`"
+      LONG_DESC = <<-LONG_DESC
+Kill a running command or cancel a scheduled command that was enqueued with `exec`.
+You can find an enqueued command's ID by running `ps`.
+
+Commands are run in an "exec" environment, separate from your webserver or worker environments.
+When they are enqueued, metadata is created in S3 and removed when the command is complete.
+Removing that metadata cancels the command.
+The command is sent a SIGTERM, and then a SIGKILL if it has not yet died.
+
+Requires AWS credentials to be set in the environment, i.e. AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY.
+      LONG_DESC
+
+      def initialize(application:, ui:)
+        @application = application
+        @ui = ui
+      end
+
+      def run(command_id)
+        ui.info("Scheduling command '#{command_id}' on #{application.name} for termination...")
+        application.kill_command(command_id)
+        ui.info("It may be a few moments before the command terminates.")
+      end
+
+      private
+
+      attr_reader :application, :ui
+    end
+  end
+end

data/lib/elastic_beans/command/ps.rb

@@ -12,6 +12,7 @@ Requires AWS credentials to be set in the environment, i.e. AWS_ACCESS_KEY_ID an
       LONG_DESC
 
       COLUMNS = {
+        "COMMAND ID" => ->(command) { command.id },
         "INSTANCE" => ->(command) { command.instance_id || SCHEDULED_INSTANCE },
         "RUN TIME" => ->(command) {
           if command.start_time
@@ -34,23 +35,14 @@ Requires AWS credentials to be set in the environment, i.e. AWS_ACCESS_KEY_ID an
       end
 
       def run
-        enqueued_commands = []
-        running_commands = []
-        ui.debug { "Fetching enqueued and running commands from #{application.name}..." }
-        threads = [
-          Thread.new do
-            enqueued_commands = application.enqueued_commands
-          end,
-          Thread.new do
-            running_commands = application.running_commands
-          end,
-        ]
-        threads.each(&:join)
+        ui.debug { "Fetching enqueued commands from #{application.name}..." }
+        enqueued_commands = application.enqueued_commands
 
+        enqueued_commands, running_commands = enqueued_commands.partition { |cmd| cmd.start_time.nil? }
         running_commands.sort_by!(&:start_time)
         commands = running_commands + enqueued_commands
         if commands.any?
-          ui.table(level: :info, columns: COLUMNS, rows: running_commands + enqueued_commands)
+          ui.table(level: :info, columns: COLUMNS, rows: commands)
         else
           ui.info("No commands enqueued or running")
         end

data/lib/elastic_beans/command.rb

@@ -2,6 +2,7 @@ require "elastic_beans/command/configure"
 require "elastic_beans/command/create"
 require "elastic_beans/command/deploy"
 require "elastic_beans/command/exec"
+require "elastic_beans/command/kill"
 require "elastic_beans/command/ps"
 require "elastic_beans/command/restart"
 require "elastic_beans/command/scale"

data/lib/elastic_beans/env_vars.rb

@@ -1,6 +1,7 @@
 require "json"
 require "aws-sdk"
 require "elastic_beans/error"
+require "elastic_beans/error/access_denied"
 
 module ElasticBeans
   # Interfaces with environment variable storage for the Elastic Beanstalk application.
@@ -24,7 +25,7 @@ module ElasticBeans
     rescue ::Aws::S3::Errors::NotFound
       {}
     rescue ::Aws::S3::Errors::Forbidden
-      raise CannotAccessConfigError.new(bucket_name: application.bucket_name, key: s3_key)
+      raise AccessDeniedS3Error.new(bucket: application.bucket_name, key: s3_key)
     end
 
     # Updates the environment variables stored in S3 by merging it with the given +env_hash+.
@@ -51,19 +52,5 @@ module ElasticBeans
     def env_script(env_hash)
       JSON.dump(env_hash)
     end
-
-    # :nodoc: all
-    # @!visibility private
-    class CannotAccessConfigError < ElasticBeans::Error
-      def initialize(bucket_name:, key:)
-        @bucket_name = bucket_name
-        @key = key
-      end
-
-      def message
-        "Cannot access configuration stored in S3 with bucket `#{@bucket_name}' and key `#{@key}'." \
-          " Please ask an administrator of your AWS account administrator to give you access."
-      end
-    end
   end
 end

data/lib/elastic_beans/error/access_denied.rb

@@ -0,0 +1,38 @@
+require "elastic_beans/error"
+
+# :nodoc: all
+# @!visibility private
+class AccessDeniedCloudFormationError < ElasticBeans::Error
+  def initialize(stack_name:)
+    @stack_name = stack_name
+  end
+
+  def message
+    <<-MESSAGE
+Access to CloudFormation stack '#{@stack_name}' was denied.
+
+Please check with your AWS administrator to give you full permission to access that stack in CloudFormation.
+    MESSAGE
+  end
+end
+
+# :nodoc: all
+# @!visibility private
+class AccessDeniedS3Error < ElasticBeans::Error
+  def initialize(bucket: nil, key: nil)
+    @bucket = bucket
+    @key = key
+  end
+
+  def message
+    msg = ""
+    if @bucket || @key
+      msg << "Access to bucket '#{@bucket}' key '#{@key}' was denied.\n"
+    else
+      msg << "Access to S3 was denied.\n"
+    end
+    msg + <<-MESSAGE
+Please check with your AWS administrator to give you full permission to access elastic_beans resources in S3.
+    MESSAGE
+  end
+end

data/lib/elastic_beans/exec/sqs_consumer.rb

@@ -6,6 +6,8 @@ require "time"
 # :nodoc: all
 # @!visibility private
 class SQSConsumer
+  LOOP_PERIOD = 1
+
   def initialize(instance_id:, logdir:, queue_url:, s3:, sqs:)
     @instance_id = instance_id
     @logdir = logdir
@@ -23,9 +25,15 @@ class SQSConsumer
     log("Starting message receive loop")
     loop do
       begin
-        sleep 1
+        sleep(LOOP_PERIOD)
         if (message = receive_message)
           command = command_from_message(message)
+          unless metadata_exists?(command: command)
+            delete_message(message)
+            log_command("Skipping command ID=#{command['id']} `#{command['command']}' because it was killed")
+            next
+          end
+
           log_command("Executing command ID=#{command['id']} `#{command['command']}' on host #{`hostname`.chomp} pid #{command_pid}...")
           start_time = Time.now
           @command_pid = Process.spawn(
@@ -35,10 +43,38 @@ class SQSConsumer
           )
           delete_message(message)
           update_metadata(command: command, start_time: start_time)
+
+          # poll command metadata to see if it is still available, if it disappears then kill the process
+          killed_thread = Thread.new do
+            loop do
+              sleep 5
+
+              if !metadata_exists?(command: command)
+                if already_killed?
+                  log_command(
+                    "Sending KILL signal to command ID=#{command['id']} `#{command['command']}' on host #{`hostname`.chomp} pid #{command_pid}" \
+                      " because it was killed and has not terminated yet"
+                  )
+                  Process.kill("KILL", command_pid)
+                  break
+                else
+                  log_command(
+                    "Sending TERM signal to command ID=#{command['id']} `#{command['command']}' on host #{`hostname`.chomp} pid #{command_pid}" \
+                      " because it was killed"
+                  )
+                  Process.kill("TERM", command_pid)
+                  @killed = true
+                end
+              end
+            end
+          end
+
           _, status = Process.wait2(command_pid)
+          killed_thread.kill if killed_thread.alive?
           log_command("Command ID=#{command['id']} `#{command['command']}' exited on host #{`hostname`.chomp}: #{status}")
 
           @command_pid = nil
+          @killed = nil
           remove_metadata(command: command)
         end
       rescue StandardError => e
@@ -89,6 +125,10 @@ class SQSConsumer
     $stderr.puts e.backtrace.join("\n")
   end
 
+  def already_killed?
+    @killed
+  end
+
   def receive_message
     sqs.receive_message(
       queue_url: queue_url,
@@ -123,6 +163,21 @@ class SQSConsumer
     log("[update_metadata] #{e.class.name}: #{e.message}\n#{e.backtrace.join("\n")}")
   end
 
+  def metadata_exists?(command:)
+    metadata = command['metadata']
+    if metadata && metadata['bucket'] && metadata['key']
+      s3.head_object(
+        bucket: metadata['bucket'],
+        key: metadata['key'],
+      )
+      true
+    end
+  rescue ::Aws::S3::Errors::NotFound
+    false
+  rescue StandardError => e
+    log("[metadata_exists] #{e.class.name}: #{e.message}\n#{e.backtrace.join("\n")}")
+  end
+
   def remove_metadata(command:)
     metadata = command['metadata']
     if metadata && metadata['bucket'] && metadata['key']

data/lib/elastic_beans/version.rb

@@ -1,3 +1,3 @@
 module ElasticBeans
-  VERSION = "0.10.0.alpha1"
+  VERSION = "0.10.0.alpha2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: elastic_beans
 version: !ruby/object:Gem::Version
-  version: 0.10.0.alpha1
+  version: 0.10.0.alpha2
 platform: ruby
 authors:
 - Adam Stegman
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-04-20 00:00:00.000000000 Z
+date: 2017-04-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -114,14 +114,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.7.0
+        version: 0.8.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.7.0
+        version: 0.8.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -173,6 +173,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".circleci/config.yml"
 - ".gitignore"
 - ".rspec"
 - Gemfile
@@ -181,7 +182,6 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
-- circle.yml
 - elastic_beans.gemspec
 - exe/beans
 - lib/elastic_beans.rb
@@ -196,6 +196,7 @@ files:
 - lib/elastic_beans/command/deploy.rb
 - lib/elastic_beans/command/exec.rb
 - lib/elastic_beans/command/get_env.rb
+- lib/elastic_beans/command/kill.rb
 - lib/elastic_beans/command/ps.rb
 - lib/elastic_beans/command/restart.rb
 - lib/elastic_beans/command/scale.rb
@@ -219,6 +220,7 @@ files:
 - lib/elastic_beans/environment/webserver.rb
 - lib/elastic_beans/environment/worker.rb
 - lib/elastic_beans/error.rb
+- lib/elastic_beans/error/access_denied.rb
 - lib/elastic_beans/error/environments_not_ready.rb
 - lib/elastic_beans/exec.rb
 - lib/elastic_beans/exec/command.rb

data/circle.yml

@@ -1,24 +0,0 @@
----
-machine:
-  environment:
-    RSPEC_OPTS: "-r rspec_junit_formatter --format progress --format RspecJunitFormatter -o $CIRCLE_TEST_REPORTS/rspec/junit.xml"
-
-dependencies:
-  override:
-    - |
-      case $CIRCLE_NODE_INDEX in
-        0)
-          rvm-exec 2.3.1 bash -c "bundle check --path=vendor/bundle_2.3 || bundle install --path=vendor/bundle_2.3 --jobs=4 --retry=3"
-          ;;
-        1)
-          rvm-exec 2.2.5 bash -c "bundle check --path=vendor/bundle_2.2 || bundle install --path=vendor/bundle_2.2 --jobs=4 --retry=3"
-          ;;
-      esac
-  post:
-  - git config --global user.name "CircleCI"
-  - git config --global user.email "ops-ro@onemedical.com"
-
-test:
-  override:
-    - case $CIRCLE_NODE_INDEX in 0) rvm-exec 2.3.1 bash -c "bundle check --path=vendor/bundle_2.3 && bundle exec rake" ;; 1) rvm-exec 2.2.5 bash -c "bundle check --path=vendor/bundle_2.2 && bundle exec rake" ;; esac:
-        parallel: true