sonic-screwdriver 1.4.0 → 2.0.0

data/lib/sonic/cli.rb

@@ -1,49 +1,59 @@
 require 'thor'
-require 'sonic/cli/help'
 
 module Sonic
-  class CLI < Command
+  class CLI < BaseCommand
+    desc "ecs SUBCOMMAND", "ecs subcommands"
+    long_desc Help.text(:ecs)
+    subcommand "ecs", Ecs
+
+    # desc "command SUBCOMMAND", "command subcommands"
+    # long_desc Help.text(:command)
+    # subcommand "command", Command
+
     class_option :verbose, type: :boolean
     class_option :noop, type: :boolean
-    class_option :cluster, desc: "ECS Cluster to use.  Default cluster is default"
-    class_option :bastion, desc: "Bastion jump host to use.  Defaults to no bastion server."
-    class_option :project_root, desc: "Project root. Useful for testing.", hide: true
-
-    desc "ssh [IDENTIFER]", "ssh into a instance using identifier. identifer can be several things: instance id, ec2 tag, ECS service name, etc"
-    long_desc Help.ssh
-    method_option :keys, :aliases => '-i', :desc => "comma separated list of ssh private key paths"
-    method_option :retry, :aliases => '-r', :type => :boolean, :desc => "keep retrying the server login until successful. Useful when on newly launched instances."
+
+    desc "ssh [IDENTIFER]", "Ssh into a instance using identifier. identifer can be several things: instance id, ec2 tag, ECS service name, etc."
+    long_desc Help.text(:ssh)
+    option :keys, :aliases => '-i', :desc => "comma separated list of ssh private key paths"
+    option :retry, :aliases => '-r', :type => :boolean, :desc => "keep retrying the server login until successful. Useful when on newly launched instances."
+    option :bastion, desc: "Bastion jump host to use.  Defaults to no bastion server."
+    option :cluster, desc: "ECS Cluster to use.  Default cluster is default"
     def ssh(identifier, *command)
       Ssh.new(identifier, options.merge(command: command)).run
     end
 
-    desc "ecs-exec [ECS_SERVICE]", "docker exec into running docker container associated with the service on a container instance"
-    long_desc Help.ecs_exec
-    def ecs_exec(service, *command)
-      Docker.new(service, options.merge(command: command)).exec
-    end
-
-    # Cannot name the command run because that is a reserved Thor keyword :(
-    desc "ecs-run [ECS_SERVICE]", "docker run with the service on a container instance"
-    long_desc Help.ecs_run
-    def ecs_run(service, *command)
-      Docker.new(service, options.merge(command: command)).run
-    end
-
-    desc "execute [FILTER] [COMMAND]", "runs command across fleet of servers via AWS Run Command"
-    long_desc Help.execute
+    desc "execute [FILTER] [COMMAND]", "Runs command across fleet of servers via AWS Run Command."
+    long_desc Help.text("execute")
     option :zero_warn, type: :boolean, default: true, desc: "Warns user when no instances found"
     # filter - Filter ec2 instances by tag name or instance_ids separated by commas
     def execute(filter, *command)
       Execute.new(command, options.merge(filter: filter)).execute
     end
 
-    desc "list [FILTER]", "lists ec2 instances"
-    long_desc Help.list
+    desc "list [FILTER]", "Lists ec2 instances."
+    long_desc Help.text(:list)
     option :header, type: :boolean, desc: "Displays header"
     # filter - Filter ec2 instances by tag name or instance_ids separated by commas
     def list(filter)
       List.new(options.merge(filter: filter)).run
     end
+
+    desc "completion *PARAMS", "Prints words for auto-completion."
+    long_desc Help.text("completion")
+    def completion(*params)
+      Completer.new(CLI, *params).run
+    end
+
+    desc "completion_script", "Generates a script that can be eval to setup auto-completion."
+    long_desc Help.text("completion_script")
+    def completion_script
+      Completer::Script.generate
+    end
+
+    desc "version", "prints version"
+    def version
+      puts VERSION
+    end
   end
 end

data/lib/sonic/command.rb

@@ -1,25 +1,11 @@
-require 'thor'
-
 module Sonic
-  class Command < Thor
-    class << self
-      def dispatch(m, args, options, config)
-        # Allow calling for help via:
-        #   sonic command help
-        #   sonic command -h
-        #   sonic command --help
-        #   sonic command -D
-        #
-        # as well thor's normal way:
-        #
-        #   sonic help command
-        help_flags = Thor::HELP_MAPPINGS + ["help"]
-        if args.length > 1 && !(args & help_flags).empty?
-          args -= help_flags
-          args.insert(-2, "help")
-        end
-        super
-      end
+  class Command < BaseCommand
+    desc "send [FILTER] [COMMAND]", "runs command across fleet of servers via AWS Run Command"
+    long_desc Help.text("command/send")
+    option :zero_warn, type: :boolean, default: true, desc: "Warns user when no instances found"
+    # filter - Filter ec2 instances by tag name or instance_ids separated by commas
+    def send(filter, *command)
+      Commander.new(command, options.merge(filter: filter)).execute
     end
   end
-end
+end

data/lib/sonic/completer.rb

@@ -0,0 +1,161 @@
+=begin
+Code Explanation:
+
+There are 3 types of things to auto-complete:
+
+  1. command: the command itself
+  2. parameters: command parameters.
+  3. options: command options
+
+Here's an example:
+
+  mycli hello name --from me
+
+  * command: hello
+  * parameters: name
+  * option: --from
+
+When command parameters are done processing, the remaining completion words will be options.  We can tell that the command params are completed based on the method arity.
+
+## Arity
+
+For example, say you had a method for a CLI command with the following form:
+
+  ufo scale service count --cluster development
+
+It's equivalent ruby method:
+
+  scale(service, count) = has an arity of 2
+
+So typing:
+
+  ufo scale service count [TAB] # there are 3 parameters including the "scale" command according to Thor's CLI processing.
+
+So the completion should only show options, something like this:
+
+  --noop --verbose --cluster
+
+## Splat Arguments
+
+When the ruby method has a splat argument, it's arity is negative.  Here are some example methods and their arities.
+
+   ship(service) = 1
+   scale(service, count) = 2
+   ships(*services) = -1
+   foo(example, *rest) = -2
+
+Fortunately, negative and positive arity values are processed the same way. So we take simply take the absolute value of the arity and process it the same.
+
+Here are some test cases, hit TAB after typing the command:
+
+  sonic completion
+  sonic completion hello
+  sonic completion hello name
+  sonic completion hello name --
+  sonic completion hello name --noop
+
+  sonic completion
+  sonic completion sub:goodbye
+  sonic completion sub:goodbye name
+
+## Subcommands and Thor::Group Registered Commands
+
+Sometimes the commands are not simple thor commands but are subcommands or Thor::Group commands. A good specific example is the ufo tool.
+
+  * regular command: ufo ship
+  * subcommand: ufo docker
+  * Thor::Group command: ufo init
+
+Auto-completion accounts for each of these type of commands.
+=end
+module Sonic
+  class Completer
+    autoload :Script, 'sonic/completer/script'
+
+    def initialize(command_class, *params)
+      @params = params
+      @current_command = @params[0]
+      @command_class = command_class # CLI initiall
+    end
+
+    def run
+      if subcommand?(@current_command)
+        subcommand_class = @command_class.subcommand_classes[@current_command]
+        @params.shift # destructive
+        Completer.new(subcommand_class, *@params).run # recursively use subcommand
+        return
+      end
+
+      # full command has been found!
+      unless found?(@current_command)
+        puts all_commands
+        return
+      end
+
+      # will only get to here if command aws found (above)
+      arity = @command_class.instance_method(@current_command).arity.abs
+      if @params.size > arity or thor_group_command?
+        puts options_completion
+      else
+        puts params_completion
+      end
+    end
+
+    def subcommand?(command)
+      @command_class.subcommands.include?(command)
+    end
+
+    # hacky way to detect that command is a registered Thor::Group command
+    def thor_group_command?
+      command_params(raw=true) == [[:rest, :args]]
+    end
+
+    def found?(command)
+      public_methods = @command_class.public_instance_methods(false)
+      command && public_methods.include?(command.to_sym)
+    end
+
+    # all top-level commands
+    def all_commands
+      commands = @command_class.all_commands.reject do |k,v|
+        v.is_a?(Thor::HiddenCommand)
+      end
+      commands.keys
+    end
+
+    def command_params(raw=false)
+      params = @command_class.instance_method(@current_command).parameters
+      # Example:
+      # >> Sub.instance_method(:goodbye).parameters
+      # => [[:req, :name]]
+      # >>
+      raw ? params : params.map!(&:last)
+    end
+
+    def params_completion
+      offset = @params.size - 1
+      offset_params = command_params[offset..-1]
+      command_params[offset..-1].first
+    end
+
+    def options_completion
+      used = ARGV.select { |a| a.include?('--') } # so we can remove used options
+
+      method_options = @command_class.all_commands[@current_command].options.keys
+      class_options = @command_class.class_options.keys
+
+      all_options = method_options + class_options + ['help']
+
+      all_options.map! { |o| "--#{o.to_s.gsub('_','-')}" }
+      filtered_options = all_options - used
+      filtered_options.uniq
+    end
+
+    # Useful for debugging. Using puts messes up completion.
+    def log(msg)
+      File.open("/tmp/complete.log", "a") do |file|
+        file.puts(msg)
+      end
+    end
+  end
+end

data/lib/sonic/completer/script.rb

@@ -0,0 +1,6 @@
+class Sonic::Completer::Script
+  def self.generate
+    bash_script = File.expand_path("script.sh", File.dirname(__FILE__))
+    puts "source #{bash_script}"
+  end
+end

data/lib/sonic/completer/script.sh

@@ -0,0 +1,10 @@
+_sonic() {
+  COMPREPLY=()
+  local word="${COMP_WORDS[COMP_CWORD]}"
+  local words=("${COMP_WORDS[@]}")
+  unset words[0]
+  local completion=$(sonic completion ${words[@]})
+  COMPREPLY=( $(compgen -W "$completion" -- "$word") )
+}
+
+complete -F _sonic sonic

data/lib/sonic/core.rb

@@ -0,0 +1,15 @@
+require 'pathname'
+require 'yaml'
+
+module Sonic
+  module Core
+    def root
+      path = ENV['SONIC_ROOT'] || '.'
+      Pathname.new(path)
+    end
+
+    def profile
+      ENV['SONIC_PROFILE'] || ENV['AWS_PROFILE']
+    end
+  end
+end

data/lib/sonic/default/settings.yml

@@ -1,16 +1,6 @@
-bastion: # cluster_host mapping
-  default: # default is nil - which means a bastion host wont be used
-  # Examples:
-  # prod: bastion.mydomain.com
-  # stag: ubuntu@bastion-stag.mydomain.com
-host_key_check: false
-service_cluster:
-  default: # defaults to nil
-  # Examples:
-  # hi-web-prod: prod
-  # hi-clock-prod: prod
-  # hi-worker-prod: prod
-  # hi-web-stag: stag
-  # hi-clock-stag: stag
-  # hi-worker-stag: stag
-user: ec2-user
+bastion:
+  host_key_check: false
+  user: ec2-user
+
+ecs_service_cluster_map:
+  default: default

data/lib/sonic/docker.rb

@@ -19,6 +19,7 @@ module Sonic
 
     def setup
       validate!
+      confirm_ssh_access
       copy_over_container_data
     end
 
@@ -46,10 +47,37 @@ module Sonic
       kernel_exec(*args)
     end
 
+    def build_host
+      host = @bastion ? bastion_host : ssh_host
+      host = "#{@user}@#{host}" unless host.include?('@')
+      host
+    end
+
+    def confirm_ssh_access
+      host = build_host
+      puts "Checking access to instance #{detector.instance_id}"
+
+      ssh = ["ssh", ssh_options, "-At", host, "uptime", "2>&1"]
+      command = ssh.join(' ')
+      puts "=> #{command}".colorize(:green)
+      output = `#{command}`
+      if output.include?("Permission denied")
+        puts output
+        UI.error("Access to the instance denied. Maybe check your ssh keys.")
+        exit 1
+      elsif output.include?(" up ")
+        puts "Access OK!"
+      else
+        puts output
+        UI.error("There was an error trying to access the instnace.")
+        exit 1
+      end
+    end
+
     def copy_over_container_data
       create_container_data
 
-      host = @bastion ? bastion_host : ssh_host
+      host = build_host
 
       # LEVEL 1
       # Always clean up remote /tmp/sonic in case of previous interrupted run.

data/lib/sonic/ecs.rb

@@ -0,0 +1,22 @@
+module Sonic
+  autoload :Docker, 'sonic/docker'
+
+  class Ecs < BaseCommand
+
+    class_option :bastion, desc: "Bastion jump host to use.  Defaults to no bastion server."
+    class_option :cluster, desc: "ECS Cluster to use.  Default cluster is default"
+
+    desc "exec [ECS_SERVICE]", "docker exec into running docker container associated with the service on a container instance"
+    long_desc Help.text("ecs/exec")
+    def exec(service, *command)
+      Docker.new(service, options.merge(command: command)).exec
+    end
+
+    # Cannot name the command run because that is a reserved Thor keyword :(
+    desc "sh [ECS_SERVICE]", "docker run with the service on a container instance"
+    long_desc Help.text("ecs/sh")
+    def sh(service, *command)
+      Docker.new(service, options.merge(command: command)).run
+    end
+  end
+end

data/lib/sonic/execute.rb

@@ -1,6 +1,10 @@
+require 'colorize'
+require 'yaml'
+require 'active_support/core_ext/hash'
+
 module Sonic
   class Execute
-    include AwsServices
+    include AwsService
 
     def initialize(command, options)
       @command = command
@@ -18,37 +22,163 @@ module Sonic
       ssm_options = build_ssm_options
       if @options[:noop]
         UI.noop = true
-        command_id = "fake command id"
+        command_id = "fake command id for noop mode"
         success = true # fake it for specs
       else
         instances_count = check_instances
         return unless instances_count > 0
 
         success = nil
+        puts "Sending command to SSM with options:"
+        puts YAML.dump(ssm_options.deep_stringify_keys)
+        puts
         begin
-          resp = ssm.send_command(ssm_options)
+          resp = send_command(ssm_options)
           command_id = resp.command.command_id
           success = true
         rescue Aws::SSM::Errors::InvalidInstanceId => e
           ssm_invalid_instance_error_message(e)
         end
       end
-      if success
-        UI.say "Command sent to AWS SSM. To check the details of the command:"
-        display_list_command(command_id)
+
+      return unless success
+
+      # IF COMMAND IS ONLY ON A SINGLE INSTANCE THEN WILL DISPLAY A BUNCH OF
+      # INFO ON THE INSTANCE. IF ITS A LOT OF INSTANCES, THEN SHOW A SUMMARY
+      # OF COMMANDS THAT WILL LEAD TO THE OUTPUT OF EACH INSTANCE.
+      UI.say "Command sent to AWS SSM. To check the details of the command:"
+      display_ssm_commands(command_id, ssm_options)
+      puts
+      return if @options[:noop]
+      wait(command_id)
+      display_ssm_output(command_id, ssm_options)
+      display_console_url(command_id)
+    end
+
+    def wait(command_id)
+      ongoing_states = ["Pending", "InProgress", "Delayed"]
+
+      print "Waiting for ssm command to finish..."
+      resp = ssm.list_commands(command_id: command_id)
+      status = resp["commands"].first["status"]
+      while ongoing_states.include?(status)
+        resp = ssm.list_commands(command_id: command_id)
+        status = resp["commands"].first["status"]
+        sleep 1
+        print '.'
+      end
+      puts "\nCommand finished."
+      puts
+    end
+
+    def display_ssm_output(command_id, ssm_options)
+      instance_ids = ssm_options[:instance_ids]
+      return unless instance_ids && instance_ids.size > 0
+
+      instance_id = instance_ids.first
+      if ssm_options[:instance_ids].size > 1
+        puts "Multiple instance targets. Only displaying output for #{instance_id}."
+      else
+        puts "Displaying output for #{instance_id}."
+      end
+
+      resp = ssm.get_command_invocation(
+        command_id: command_id, instance_id: instance_id
+      )
+      puts "Command status: #{colorized_status(resp["status"])}"
+      ssm_output(resp, "output")
+      ssm_output(resp, "error")
+      puts
+    end
+
+    def display_console_url(command_id)
+      region = `aws configure get region`.strip rescue 'us-east-1'
+      console_url = "https://#{region}.console.aws.amazon.com/systems-manager/run-command/#{command_id}"
+      puts "To see the more output details visit:"
+      puts "  #{console_url}"
+      puts
+      copy_paste_clipboard(console_url)
+      UI.say "Pro tip: the console url is already in your copy/paste clipboard."
+    end
+
+    def colorized_status(status)
+      case status
+      when "Success"
+        status.colorize(:green)
+      when "Failed"
+        status.colorize(:red)
+      else
+        status
+      end
+    end
+
+    # type: output or error
+    def ssm_output(resp, type)
+      content_key = "standard_#{type}_content"
+      s3_key = "standard_#{type}_url"
+
+      content = resp[content_key]
+      return if content.empty?
+
+      puts "Command standard #{type}:"
+      # "https://s3.amazonaws.com/lr-infrastructure-prod/ssm/commands/sonic/0a4f4bef-8f63-4235-8b30-ae296477261a/i-0b2e6e187a3f9ada9/awsrunPowerShellScript/0.awsrunPowerShellScript/stderr">
+      if content.include?("--output truncated--") && !resp[s3_key].empty?
+        s3_url = resp[s3_key]
+        info = s3_url.sub('https://s3.amazonaws.com/', '').split('/')
+        bucket = info[0]
+        key = info[1..-1].join('/')
+        resp = s3.get_object(bucket: bucket, key: key)
+        data = resp.body.read
+        puts data
+
+        path = "/tmp/sonic-output.txt"
+        puts "------"
+        puts "Output also written to #{path}"
+        IO.write(path, data)
+      else
+        puts content
       end
+
+      # puts "#{s3_key}: #{resp[s3_key]}"
+    end
+
+    def send_command(options)
+      retries = 0
+
+      begin
+        resp = ssm.send_command(options)
+        # puts "NOOP FOR NOW"
+      rescue Aws::SSM::Errors::UnsupportedPlatformType
+        retries += 1
+        # toggle AWS-RunShellScript / AWS-RunPowerShellScript
+        options[:document_name] =
+          options[:document_name] == "AWS-RunShellScript" ?
+          "AWS-RunPowerShellScript" : "AWS-RunShellScript"
+
+        puts "#{$!}"
+        puts "Retrying with document_name #{options[:document_name]}"
+        puts "Retries: #{retries}"
+
+        retries <= 1 ? retry : raise
+      end
+
+      resp
     end
 
     def build_ssm_options
       criteria = transform_filter(@filter)
       command = build_command(@command)
-      criteria.merge(
-        document_name: "AWS-RunShellScript",
-        comment: "sonic #{ARGV.join(' ')}",
-        parameters: {
-          "commands" => command
-        }
+      options = criteria.merge(
+        document_name: "AWS-RunShellScript", # default
+        comment: "sonic #{ARGV.join(' ')}"[0..99], # comment has a max of 100 chars
+        parameters: { "commands" => command }
       )
+      settings_options = settings["send_command"] || {}
+      options.merge(settings_options.deep_symbolize_keys)
+    end
+
+    def settings
+      @settings ||= Setting.new.data
     end
 
     #
@@ -125,6 +255,7 @@ You can use the following command to check registered instances to SSM.
       EOS
       UI.warn(message)
       copy_paste_clipboard(ssm_describe_command)
+      UI.say "Pro tip: ssm describe-instance-information already in your copy/paste clipboard."
     end
 
     def file_path?(command)
@@ -136,7 +267,7 @@ You can use the following command to check registered instances to SSM.
     def file_path(command)
       path = command.first
       path = path.sub('file://', '')
-      path = "#{@options[:project_root]}/#{path}" if @options[:project_root]
+      path = "#{Sonic.root}/#{path}"
       path
     end
 
@@ -150,7 +281,7 @@ You can use the following command to check registered instances to SSM.
       instances = List.new(@options).instances
       if instances.count == 0
         message = <<-EOL
-  Unable to find any instances with filter #{@filter.join(',')}.
+Unable to find any instances with filter #{@filter.join(',')}.
   Are you sure you specify the filter with either a EC2 tag or list instance ids?
   If you are using ECS identifiers, they are not supported with this command.
 EOL
@@ -170,16 +301,20 @@ EOL
       text =~ /i-.{17}/ || text =~ /i-.{8}/
     end
 
-    def display_list_command(command_id)
-      list_command = "aws ssm list-commands --command-id #{command_id}"
+    def display_ssm_commands(command_id, ssm_options)
+      list_command = "  aws ssm list-commands --command-id #{command_id}"
       UI.say list_command
-      copy_paste_clipboard(list_command)
+
+      return unless ssm_options[:instance_ids]
+      ssm_options[:instance_ids].each do |instance_id|
+        get_command = "  aws ssm get-command-invocation --command-id #{command_id} --instance-id #{instance_id}"
+        UI.say get_command
+      end
     end
 
     def copy_paste_clipboard(command)
       return unless RUBY_PLATFORM =~ /darwin/
       system("echo '#{command}' | pbcopy")
-      UI.say "Pro tip: the aws ssm command is already in your copy/paste clipboard."
     end
   end
 end