elastic_beans 0.10.0.alpha6 → 0.10.0.alpha7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4aa4fc0dc734f2eeee83efeed1dd7b288f3f9904
-  data.tar.gz: 703e6b50540326e18a48ad9abb79da92e940a8c3
+  metadata.gz: 2fec2d342af0cabded810ab2d37bf6f810b52a67
+  data.tar.gz: 6c2c698d6b72232de2461ec76f96084488a4e8f6
 SHA512:
-  metadata.gz: f6cbb95eb184b7a85f247c45062dc3a5f2bebf2af8480bd32802f698bcc2fdcaa084fb8ecf12ef4f1b072cc313c76fcdaf610d3a6caf9b03816cc4cf54a0006e
-  data.tar.gz: 48648ab0283f6130b6d54a611ae36034a818e3625510893384ee74bf77c189154137c6d1a48c1674be3b7e71bad8b03904b55171f52e95669e3306cfe73c4ec0
+  metadata.gz: 3b0b71d95a2282b86822c3675246e62604ff488f7ff17db594899d82593455b09782dadad20a5d3a01daab80992ffc5b58c52832d8ecc17dab69ebd5dee0a571
+  data.tar.gz: d9213516af9141f51372bbe55a57067dc28b2ab8f186a9b4c0e281ef3184427e80c7683c2c0e65541fd1fc28b8fbab93e86d0eae38d6a6001a3a2d3068cc3429

data/README.md

@@ -51,12 +51,16 @@ As the SDK documentation suggests, using environment variables is recommended.
     beans deploy -a myapp
 
     # Run one-off tasks
-    beans exec -a myapp rake db:migrate
+    beans exec rake db:migrate -a myapp
 
     # SSH to an instance for debugging, tunneling through a bastion instance to reach the private network
     beans ssh -a myapp webserver [-n INDEX] [-i IDENTITY_FILE] [-u USERNAME] \
       [-b BASTION_HOST] [--bastion-identity-file BASTION_IDENTITY_FILE] [--bastion-username BASTION_USERNAME]
 
+    # Run interactive commands via SSH
+    beans exec rails console -a myapp --interactive [-i IDENTITY_FILE] [-u USERNAME] \
+      [-b BASTION_HOST] [--bastion-identity-file BASTION_IDENTITY_FILE] [--bastion-username BASTION_USERNAME]
+
     # Update all existing environments and configuration
     beans configure -n myapp-networking -a myapp \
       [-b SECRET_KEY_BASE] [-d DATABASE_URL] [-k KEYPAIR] \

data/lib/elastic_beans/application.rb

@@ -107,7 +107,7 @@ module ElasticBeans
     # Raises an error if the exec environment or queue cannot be found.
     def enqueue_command(command)
       if environments.none? { |environment| environment.is_a?(Environment::Exec) }
-        raise MissingExecEnvironmentError
+        raise MissingExecEnvironmentError.new(application: self)
       end
 
       if command.to_s == command
@@ -269,11 +269,15 @@ module ElasticBeans
     # :nodoc: all
     # @!visibility private
     class MissingExecEnvironmentError < ElasticBeans::Error
+      def initialize(application:)
+        @application = application
+      end
+
       def message
         <<-MESSAGE
 A one-off command cannot be executed because the "exec" environment does not exist. Please create it:
 
-    #{command_as_string "-a APPLICATION exec"}
+    #{command_as_string "create -a #{@application.name} exec"}
         MESSAGE
       end
     end

data/lib/elastic_beans/cli.rb

@@ -86,12 +86,25 @@ class ElasticBeans::CLI < Thor
   desc ElasticBeans::Command::Exec::USAGE, ElasticBeans::Command::Exec::DESC
   long_desc ElasticBeans::Command::Exec::LONG_DESC
   option :application, aliases: %w(-a), required: true, desc: APPLICATION_DESC
+  option :interactive, aliases: %w(-t), type: :boolean, desc: "Run this command interactively, via SSH. Set SSH options as necessary."
+  option :bastion_host, aliases: %w(-b), desc: "The hostname of the bastion server in the VPC"
+  option :bastion_identity_file, desc: "The SSH private key to use to connect to the bastion server"
+  option :bastion_username, desc: "The username to use to connect to the bastion server"
+  option :identity_file, aliases: %w(-i), desc: "The SSH private key associated with the keypair used when creating the environment"
+  option :username, aliases: %w(-u), desc: "The username to log into the instance with, e.g. `ec2-user`"
   def exec(*command_parts)
     @verbose = options[:verbose]
     ElasticBeans::Command::Exec.new(
       application: application(
         name: options[:application],
       ),
+      bastion_host: options[:bastion_host],
+      bastion_identity_file: options[:bastion_identity_file],
+      bastion_username: options[:bastion_username],
+      identity_file: options[:identity_file],
+      interactive: options[:interactive],
+      username: options[:username],
+      ec2: ec2_client,
       ui: ui,
     ).run(*command_parts)
   rescue StandardError => e
@@ -215,8 +228,8 @@ class ElasticBeans::CLI < Thor
 
   map ["delvar", "rmvar"] => "unsetenv"
 
-  desc ElasticBeans::Command::Ssh::USAGE, ElasticBeans::Command::Ssh::DESC
-  long_desc ElasticBeans::Command::Ssh::LONG_DESC
+  desc ElasticBeans::Command::SSH::USAGE, ElasticBeans::Command::SSH::DESC
+  long_desc ElasticBeans::Command::SSH::LONG_DESC
   option :application, aliases: %w(-a), required: true, desc: APPLICATION_DESC
   option :bastion_host, aliases: %w(-b), desc: "The hostname of the bastion server in the VPC"
   option :bastion_identity_file, desc: "The SSH private key to use to connect to the bastion server"
@@ -227,7 +240,7 @@ class ElasticBeans::CLI < Thor
   option :username, aliases: %w(-u), desc: "The username to log into the instance with, e.g. `ec2-user`"
   def ssh(environment_type)
     @verbose = options[:verbose]
-    ElasticBeans::Command::Ssh.new(
+    ElasticBeans::Command::SSH.new(
       application: application(
         name: options[:application],
       ),

data/lib/elastic_beans/command/exec.rb

@@ -12,28 +12,186 @@ The command is run in an "exec" environment, separate from your webserver or wor
 You must create the exec environment prior to this command being run: `beans create exec -a APPLICATION`.
 Output from the command is appended to /var/log/elastic_beans/exec/command.log.
 
+If interactive, runs the command via SSH, tunneling through a bastion server if specified.
+
 Requires AWS credentials to be set in the environment, i.e. AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY.
       LONG_DESC
 
-      def initialize(application:, ui:)
+      # :category: internal
+      COMMAND_LOGFILE = "/var/log/elastic_beans/exec/command.log"
+      # :category: internal
+      COMMAND_SCRIPT = "/opt/elastic_beans/exec/run_command.sh"
+      # :category: internal
+      TAKEN_WAIT_PERIOD = 5
+      # :category: internal
+      TAKEN_WAIT_TIMEOUT = 120
+
+      def initialize(
+        application:,
+        bastion_host:,
+        bastion_identity_file:,
+        bastion_username:,
+        identity_file:,
+        interactive:,
+        username:,
+        ec2:,
+        ui:
+      )
         @application = application
+        @bastion_host = bastion_host
+        @bastion_identity_file = bastion_identity_file
+        @bastion_username = bastion_username
+        @identity_file = identity_file
+        @interactive = interactive
+        @username = username || "ec2-user"
+        @ec2 = ec2
         @ui = ui
       end
 
       def run(*command_parts)
-        command = command(command_parts)
+        command = command_from_parts(command_parts)
         ui.info("Running `#{command.command_string}' on #{application.name}... (ID=#{command.id})")
-        application.enqueue_command(command)
+
+        if interactive?
+          ui.info("Finding an exec instance...")
+          freeze_command = ::ElasticBeans::Exec::Command.freeze_instance
+
+          begin
+            ui.debug { "Freezing exec instance... (ID=#{freeze_command.id})" }
+            application.enqueue_command(freeze_command)
+            freeze_command = wait_until_command_taken(freeze_command)
+            ui.debug { "Frozen exec instance: '#{freeze_command.instance_id}'" }
+
+            begin
+              instance_ip = ec2.describe_instances(instance_ids: [freeze_command.instance_id]).reservations[0].instances[0].private_ip_address
+            rescue ::Aws::EC2::Errors::InvalidInstanceIDMalformed
+              raise TerminatedInstanceError.new(instance_id: freeze_command.instance_id)
+            end
+            # It's possible a retry would just work, but I'd like to see this happen in reality before I assume that.
+            if instance_ip.nil?
+              raise TerminatedInstanceError.new(instance_id: freeze_command.instance_id)
+            end
+
+            ui.info("Connecting to #{username}@#{instance_ip}...")
+            ElasticBeans::SSH.new(
+              hostname: instance_ip,
+              username: username,
+              identity_file: identity_file,
+              bastion_host: bastion_host,
+              bastion_username: bastion_username,
+              bastion_identity_file: bastion_identity_file,
+              ssh_options: %w(-t),
+              command: [
+                # Do not lose command exit status
+                *%w(set -o pipefail &&),
+                # Log command start just like SQSConsumer
+                "sudo", "echo", %("Executing command ID=#{command.id} \\`#{command.command_string}' on host `hostname` pid $$..."),
+                  ">>", COMMAND_LOGFILE, "&&",
+                # Load application environment
+                "sudo", COMMAND_SCRIPT, *command_parts,
+                # Log command execution
+                  "|", "tee", "-a", COMMAND_LOGFILE, ";",
+                # Save command exit status for final exit
+                *%w(status=$? ;),
+                # Log command end just like SQSConsumer
+                "sudo", "echo", %("Command ID=#{command.id} \\`#{command.command_string}' exited on host `hostname`: pid $$ exit $status"),
+                  ">>", COMMAND_LOGFILE, "&&",
+                # Propagate command exit status
+                *%w(exit $status),
+              ],
+              logger: ui,
+            ).connect
+          rescue ElasticBeans::SSH::BastionAuthenticationError => e
+            raise BastionAuthenticationError.new(cause: e)
+          ensure
+            application.kill_command(freeze_command)
+          end
+        else
+          application.enqueue_command(command)
+        end
       end
 
       private
 
-      attr_reader :application, :ui
+      attr_reader(
+        :application,
+        :bastion_host,
+        :bastion_identity_file,
+        :bastion_username,
+        :identity_file,
+        :username,
+        :ec2,
+        :ui,
+      )
 
-      def command(command_parts)
+      def command_from_parts(command_parts)
         command_string = command_parts.map { |word| Shellwords.escape(word) }.join(" ")
         ::ElasticBeans::Exec::Command.new(command_string: command_string)
       end
+
+      def interactive?
+        @interactive
+      end
+
+      def wait_until_command_taken(command)
+        taken_command = nil
+        Timeout.timeout(TAKEN_WAIT_TIMEOUT) do
+          until taken_command
+            sleep(TAKEN_WAIT_PERIOD)
+            taken_command = application.enqueued_commands.find { |cmd| cmd == command && !cmd.instance_id.nil? }
+          end
+        end
+        taken_command
+      rescue Timeout::Error
+        raise ExecEnvironmentBusyError.new(timeout: TAKEN_WAIT_TIMEOUT)
+      end
+
+      class BastionAuthenticationError < ElasticBeans::Error
+        def initialize(cause:)
+          @cause = cause
+        end
+
+        def backtrace
+          @cause.backtrace
+        end
+
+        def message
+          <<-MESSAGE
+#{@cause.message}
+
+Please check the bastion options and try again.
+
+#{command_help "exec"}
+          MESSAGE
+        end
+      end
+
+      class ExecEnvironmentBusyError < ElasticBeans::Error
+        def initialize(timeout:)
+          @timeout = timeout
+        end
+
+        def message
+          <<-MESSAGE
+The exec environment is too busy to reserve an instance for this interactive session (we waited #{@timeout} seconds).
+Please scale it up and try again.
+          MESSAGE
+        end
+      end
+
+      class TerminatedInstanceError < ElasticBeans::Error
+        def initialize(instance_id:)
+          @instance_id = instance_id
+        end
+
+        def message
+          <<-MESSAGE
+The exec instance '#{@instance_id}' was terminated.
+Please check the exec environment status, wait for the environment to stabilize, and try again.
+          MESSAGE
+          msg
+        end
+      end
     end
   end
 end

data/lib/elastic_beans/command/ps.rb

@@ -37,6 +37,7 @@ Requires AWS credentials to be set in the environment, i.e. AWS_ACCESS_KEY_ID an
       def run
         ui.debug { "Fetching enqueued commands from #{application.name}..." }
         enqueued_commands = application.enqueued_commands
+        enqueued_commands.reject!(&:freeze_instance?)
 
         enqueued_commands, running_commands = enqueued_commands.partition { |cmd| cmd.start_time.nil? }
         running_commands.sort_by!(&:start_time)

data/lib/elastic_beans/command/ssh.rb

@@ -4,7 +4,7 @@ require "elastic_beans/error/environments_not_ready"
 module ElasticBeans
   module Command
     # :nodoc: all
-    class Ssh
+    class SSH
       USAGE = "ssh ENVIRONMENT_TYPE"
       DESC = "Connect to an instance for debugging, tunneling through a bastion server"
       LONG_DESC = <<-LONG_DESC
@@ -53,57 +53,26 @@ Requires AWS credentials to be set in the environment, i.e. AWS_ACCESS_KEY_ID an
           raise NoInstanceError.new(environment: environment, index: index)
         end
 
-        begin
-          instance_ip = ec2.describe_instances(instance_ids: [instance_id]).reservations[0].instances[0].private_ip_address
-        rescue ::Aws::EC2::Errors::InvalidInstanceIDMalformed
-          raise NoInstanceError.new(environment: environment, index: index)
-        end
+        instance_ip = ec2.describe_instances(instance_ids: [instance_id]).reservations[0].instances[0].private_ip_address
         # It's possible a retry would just work, but I'd like to see this happen in reality before I assume that.
         if instance_ip.nil?
           raise TerminatedInstanceError.new(instance_id: instance_id, environment: environment)
         end
 
-        ssh_args = ["ssh"]
-
-        if identity_file
-          ssh_args << "-i"
-          ssh_args << identity_file
-        end
-
-        gateway = nil
-        if bastion_host
-          ssh_args << "#{username}@localhost"
-          ssh_args += ["-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no"]
-          begin
-            bastion_options = {auth_methods: ["publickey"]}
-            if bastion_identity_file
-              bastion_options[:key_data] = File.read(bastion_identity_file)
-            end
-
-            ui.info("Connecting to '#{bastion_username}@#{bastion_host}'...")
-            gateway = Net::SSH::Gateway.new(
-              bastion_host,
-              bastion_username,
-              bastion_options
-            )
-            port = gateway.open(instance_ip, 22)
-            ui.debug { "Opened gateway to '#{instance_ip}' on port '#{port}'" }
-            ssh_args += ["-p", port.to_s]
-          rescue Net::SSH::AuthenticationFailed => e
-            raise BastionAuthenticationError.new(cause: e)
-          end
-        else
-          ssh_args << "#{username}@#{instance_ip}"
-        end
-
-        ui.info("Connecting to '#{username}@#{instance_ip}'...")
-        pid = Process.spawn(*ssh_args)
-        _, status = Process.waitpid2(pid)
-        unless status.success?
-          raise SshFailedError
-        end
-      ensure
-        gateway.close(port) if gateway
+        ui.info("Connecting to #{username}@#{instance_ip}...")
+        ElasticBeans::SSH.new(
+          hostname: instance_ip,
+          username: username,
+          identity_file: identity_file,
+          bastion_host: bastion_host,
+          bastion_username: bastion_username,
+          bastion_identity_file: bastion_identity_file,
+          logger: ui,
+        ).connect
+      rescue ::Aws::EC2::Errors::InvalidInstanceIDMalformed
+        raise NoInstanceError.new(environment: environment, index: index)
+      rescue ElasticBeans::SSH::BastionAuthenticationError => e
+        raise BastionAuthenticationError.new(cause: e)
       end
 
       private
@@ -127,11 +96,15 @@ Requires AWS credentials to be set in the environment, i.e. AWS_ACCESS_KEY_ID an
           @cause = cause
         end
 
+        def backtrace
+          @cause.backtrace
+        end
+
         def message
           <<-MESSAGE
 #{@cause.message}
 
-Please check the --bastion-identity-file or --bastion-username options and try again.
+Please check the bastion options and try again.
 
 #{command_help "ssh"}
           MESSAGE
@@ -159,12 +132,6 @@ Please check the --bastion-identity-file or --bastion-username options and try a
         end
       end
 
-      class SshFailedError < ElasticBeans::Error
-        def message
-          ""
-        end
-      end
-
       class TerminatedInstanceError < ElasticBeans::Error
         def initialize(instance_id:, environment:)
           @instance_id = instance_id
@@ -175,8 +142,6 @@ Please check the --bastion-identity-file or --bastion-username options and try a
           <<-MESSAGE
 The instance '#{@instance_id}' in the environment '#{@environment.name}' has been terminated.
 Please try again in a moment.
-
-#{command_help "ssh"}
           MESSAGE
         end
       end

data/lib/elastic_beans/environment.rb

@@ -21,6 +21,10 @@ module ElasticBeans
     TIER_TYPE = "Standard"
     # :category: Internal
     WORKER_TEMPLATE_NAME_PATTERN = /worker-(?<queue>\w+)/
+    # :category: Internal
+    WAIT_TIMEOUT = 3600
+    # :category: Internal
+    DEGRADED_TIMEOUT = 90
 
     attr_reader :name
 
@@ -279,7 +283,7 @@ module ElasticBeans
       status = wait_status[0]
       health = "Grey"
       health_status = wait_health_status[0]
-      Timeout.timeout(3600) do
+      Timeout.timeout(WAIT_TIMEOUT) do
         while wait_status.include?(status) || wait_health_status.include?(health_status)
           sleep 5
           environment = environment_description
@@ -289,8 +293,8 @@ module ElasticBeans
         end
       end
 
-      # Wait a little bit longer. Sometimes apps are Ready but still Degraded for a few seconds.
-      Timeout.timeout(30) do
+      # Wait a little bit longer. Sometimes apps are Ready but still Degraded for a few moments.
+      Timeout.timeout(DEGRADED_TIMEOUT) do
         while status == "Ready" && health != "Green"
           sleep 5
           environment = environment_description

data/lib/elastic_beans/exec/ebextension.yml

@@ -1,8 +1,12 @@
 container_commands:
   00_copy_files:
-    command: "mkdir -vp /opt/elastic_beans && cp -vR /var/app/ondeck/.elastic_beans/exec /opt/elastic_beans/"
+    command: "mkdir -vp /opt/elastic_beans && cp -vR .elastic_beans/exec /opt/elastic_beans/"
   01_permissions:
     command: "chmod 755 /opt/elastic_beans/exec/run_command.sh"
+  05_log_files:
+    command: |
+      mkdir -vp /var/log/elastic_beans/exec
+      touch /var/log/elastic_beans/exec/command.log
   09_logrotate:
     command: "cp -v /opt/elastic_beans/exec/logrotate /etc/logrotate.d/elastic_beans_exec"
     test: "/opt/elasticbeanstalk/bin/get-config meta -k sqsdconfig --output YAML | grep -q '^environment_name: .*-exec$'"

data/lib/elastic_beans/exec/run_command.sh

@@ -1,5 +1,7 @@
 #!/bin/sh
 
+set -e
+
 # execute in the context of the application
 EB_APP_DIR="`/opt/elasticbeanstalk/bin/get-config container -k app_deploy_dir`"
 cd "$EB_APP_DIR"

data/lib/elastic_beans/scheduler/ebextension.yml

@@ -1,4 +1,4 @@
 container_commands:
   remove_scheduler_file:
-    command: "rm -f /var/app/ondeck/cron.yaml"
+    command: "rm -f cron.yaml"
     test: "/opt/elasticbeanstalk/bin/get-config meta -k sqsdconfig --output YAML | grep '^environment_name: ' | grep -qv '^environment_name: .*-scheduler$'"

data/lib/elastic_beans/ssh.rb

@@ -0,0 +1,110 @@
+require 'logger'
+
+module ElasticBeans
+  # Opens an SSH connection, tunneling through a bastion host if specified.
+  class SSH
+    def initialize(
+      hostname:,
+      username:,
+      bastion_host: nil,
+      bastion_username: nil,
+      bastion_identity_file: nil,
+      identity_file: nil,
+      ssh_options: [],
+      command: [],
+      logger: Logger.new('/dev/null')
+    )
+      @hostname = hostname
+      @username = username
+      @bastion_host = bastion_host
+      @bastion_username = bastion_username
+      @bastion_identity_file = bastion_identity_file
+      @identity_file = identity_file
+      @ssh_options = ssh_options
+      @command = command
+      @logger = logger
+    end
+
+    # Opens an SSH connection, tunneling through a bastion host if specified.
+    # Spawns a child process using the ssh command-line utility and waits for it to complete.
+    #
+    # Raises an error if SSH exits with a non-zero exit status.
+    def connect
+      ssh_args = ["ssh", *ssh_options]
+
+      if identity_file
+        ssh_args << "-i"
+        ssh_args << identity_file
+      end
+
+      gateway = nil
+      if bastion_host
+        ssh_args << "#{username}@localhost"
+        ssh_args += ["-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no"]
+        begin
+          bastion_options = {auth_methods: ["publickey"]}
+          if bastion_identity_file
+            bastion_options[:key_data] = File.read(bastion_identity_file)
+          end
+
+          logger.info("Connecting to '#{bastion_username}@#{bastion_host}'...")
+          gateway = Net::SSH::Gateway.new(
+            bastion_host,
+            bastion_username,
+            bastion_options,
+          )
+          port = gateway.open(hostname, 22)
+          logger.debug { "Opened gateway to '#{hostname}' on port '#{port}'" }
+          ssh_args += ["-p", port.to_s]
+        rescue Net::SSH::Exception => e
+          raise BastionAuthenticationError.new(cause: e)
+        end
+      else
+        ssh_args << "#{username}@#{hostname}"
+      end
+
+      logger.debug { "Connecting: `#{ssh_args.join(' ')}'..." }
+      pid = Process.spawn(*ssh_args, *command)
+      _, status = Process.waitpid2(pid)
+      unless status.success?
+        raise SSHFailedError
+      end
+    ensure
+      gateway.close(port) if gateway
+    end
+
+    private
+
+    attr_reader(
+      :bastion_host,
+      :bastion_identity_file,
+      :bastion_username,
+      :command,
+      :identity_file,
+      :hostname,
+      :ssh_options,
+      :username,
+      :logger,
+    )
+
+    class BastionAuthenticationError < ElasticBeans::Error
+      def initialize(cause:)
+        @cause = cause
+      end
+
+      def backtrace
+        @cause.backtrace
+      end
+
+      def message
+        @cause.message
+      end
+    end
+
+    class SSHFailedError < ElasticBeans::Error
+      def message
+        ""
+      end
+    end
+  end
+end

data/lib/elastic_beans/version.rb

@@ -1,3 +1,3 @@
 module ElasticBeans
-  VERSION = "0.10.0.alpha6"
+  VERSION = "0.10.0.alpha7"
 end

data/lib/elastic_beans.rb

@@ -8,4 +8,5 @@ require "elastic_beans/env_vars"
 require "elastic_beans/environment"
 require "elastic_beans/exec"
 require "elastic_beans/network"
+require "elastic_beans/ssh"
 require "elastic_beans/version"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: elastic_beans
 version: !ruby/object:Gem::Version
-  version: 0.10.0.alpha6
+  version: 0.10.0.alpha7
 platform: ruby
 authors:
 - Adam Stegman
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-04-26 00:00:00.000000000 Z
+date: 2017-04-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -233,6 +233,7 @@ files:
 - lib/elastic_beans/network.rb
 - lib/elastic_beans/rack/exec.rb
 - lib/elastic_beans/scheduler/ebextension.yml
+- lib/elastic_beans/ssh.rb
 - lib/elastic_beans/ui.rb
 - lib/elastic_beans/version.rb
 homepage: https://github.com/onemedical/elastic_beans