sonic-screwdriver 1.4.0 → 2.2.1

data/lib/sonic/command.rb

@@ -1,25 +1,11 @@
-require 'thor'
-
 module Sonic
-  class Command < Thor
-    class << self
-      def dispatch(m, args, options, config)
-        # Allow calling for help via:
-        #   sonic command help
-        #   sonic command -h
-        #   sonic command --help
-        #   sonic command -D
-        #
-        # as well thor's normal way:
-        #
-        #   sonic help command
-        help_flags = Thor::HELP_MAPPINGS + ["help"]
-        if args.length > 1 && !(args & help_flags).empty?
-          args -= help_flags
-          args.insert(-2, "help")
-        end
-        super
-      end
+  class Command < BaseCommand
+    desc "send [FILTER] [COMMAND]", "runs command across fleet of servers via AWS Run Command"
+    long_desc Help.text("command/send")
+    option :zero_warn, type: :boolean, default: true, desc: "Warns user when no instances found"
+    # filter - Filter ec2 instances by tag name or instance_ids separated by commas
+    def send(filter, *command)
+      Commander.new(command, options.merge(filter: filter)).execute
     end
   end
-end
+end

data/lib/sonic/completer.rb

@@ -0,0 +1,161 @@
+=begin
+Code Explanation:
+
+There are 3 types of things to auto-complete:
+
+  1. command: the command itself
+  2. parameters: command parameters.
+  3. options: command options
+
+Here's an example:
+
+  mycli hello name --from me
+
+  * command: hello
+  * parameters: name
+  * option: --from
+
+When command parameters are done processing, the remaining completion words will be options.  We can tell that the command params are completed based on the method arity.
+
+## Arity
+
+For example, say you had a method for a CLI command with the following form:
+
+  ufo scale service count --cluster development
+
+It's equivalent ruby method:
+
+  scale(service, count) = has an arity of 2
+
+So typing:
+
+  ufo scale service count [TAB] # there are 3 parameters including the "scale" command according to Thor's CLI processing.
+
+So the completion should only show options, something like this:
+
+  --noop --verbose --cluster
+
+## Splat Arguments
+
+When the ruby method has a splat argument, it's arity is negative.  Here are some example methods and their arities.
+
+   ship(service) = 1
+   scale(service, count) = 2
+   ships(*services) = -1
+   foo(example, *rest) = -2
+
+Fortunately, negative and positive arity values are processed the same way. So we take simply take the absolute value of the arity and process it the same.
+
+Here are some test cases, hit TAB after typing the command:
+
+  sonic completion
+  sonic completion hello
+  sonic completion hello name
+  sonic completion hello name --
+  sonic completion hello name --noop
+
+  sonic completion
+  sonic completion sub:goodbye
+  sonic completion sub:goodbye name
+
+## Subcommands and Thor::Group Registered Commands
+
+Sometimes the commands are not simple thor commands but are subcommands or Thor::Group commands. A good specific example is the ufo tool.
+
+  * regular command: ufo ship
+  * subcommand: ufo docker
+  * Thor::Group command: ufo init
+
+Auto-completion accounts for each of these type of commands.
+=end
+module Sonic
+  class Completer
+    autoload :Script, 'sonic/completer/script'
+
+    def initialize(command_class, *params)
+      @params = params
+      @current_command = @params[0]
+      @command_class = command_class # CLI initiall
+    end
+
+    def run
+      if subcommand?(@current_command)
+        subcommand_class = @command_class.subcommand_classes[@current_command]
+        @params.shift # destructive
+        Completer.new(subcommand_class, *@params).run # recursively use subcommand
+        return
+      end
+
+      # full command has been found!
+      unless found?(@current_command)
+        puts all_commands
+        return
+      end
+
+      # will only get to here if command aws found (above)
+      arity = @command_class.instance_method(@current_command).arity.abs
+      if @params.size > arity or thor_group_command?
+        puts options_completion
+      else
+        puts params_completion
+      end
+    end
+
+    def subcommand?(command)
+      @command_class.subcommands.include?(command)
+    end
+
+    # hacky way to detect that command is a registered Thor::Group command
+    def thor_group_command?
+      command_params(raw=true) == [[:rest, :args]]
+    end
+
+    def found?(command)
+      public_methods = @command_class.public_instance_methods(false)
+      command && public_methods.include?(command.to_sym)
+    end
+
+    # all top-level commands
+    def all_commands
+      commands = @command_class.all_commands.reject do |k,v|
+        v.is_a?(Thor::HiddenCommand)
+      end
+      commands.keys
+    end
+
+    def command_params(raw=false)
+      params = @command_class.instance_method(@current_command).parameters
+      # Example:
+      # >> Sub.instance_method(:goodbye).parameters
+      # => [[:req, :name]]
+      # >>
+      raw ? params : params.map!(&:last)
+    end
+
+    def params_completion
+      offset = @params.size - 1
+      offset_params = command_params[offset..-1]
+      command_params[offset..-1].first
+    end
+
+    def options_completion
+      used = ARGV.select { |a| a.include?('--') } # so we can remove used options
+
+      method_options = @command_class.all_commands[@current_command].options.keys
+      class_options = @command_class.class_options.keys
+
+      all_options = method_options + class_options + ['help']
+
+      all_options.map! { |o| "--#{o.to_s.gsub('_','-')}" }
+      filtered_options = all_options - used
+      filtered_options.uniq
+    end
+
+    # Useful for debugging. Using puts messes up completion.
+    def log(msg)
+      File.open("/tmp/complete.log", "a") do |file|
+        file.puts(msg)
+      end
+    end
+  end
+end

data/lib/sonic/completer/script.rb

@@ -0,0 +1,6 @@
+class Sonic::Completer::Script
+  def self.generate
+    bash_script = File.expand_path("script.sh", File.dirname(__FILE__))
+    puts "source #{bash_script}"
+  end
+end

data/lib/sonic/completer/script.sh

@@ -0,0 +1,10 @@
+_sonic() {
+  COMPREPLY=()
+  local word="${COMP_WORDS[COMP_CWORD]}"
+  local words=("${COMP_WORDS[@]}")
+  unset words[0]
+  local completion=$(sonic completion ${words[@]})
+  COMPREPLY=( $(compgen -W "$completion" -- "$word") )
+}
+
+complete -F _sonic sonic

data/lib/sonic/core.rb

@@ -0,0 +1,15 @@
+require 'pathname'
+require 'yaml'
+
+module Sonic
+  module Core
+    def root
+      path = ENV['SONIC_ROOT'] || '.'
+      Pathname.new(path)
+    end
+
+    def profile
+      ENV['SONIC_PROFILE'] || ENV['AWS_PROFILE']
+    end
+  end
+end

data/lib/sonic/default/settings.yml

@@ -1,16 +1,9 @@
-bastion: # cluster_host mapping
-  default: # default is nil - which means a bastion host wont be used
-  # Examples:
-  # prod: bastion.mydomain.com
-  # stag: ubuntu@bastion-stag.mydomain.com
-host_key_check: false
-service_cluster:
-  default: # defaults to nil
-  # Examples:
-  # hi-web-prod: prod
-  # hi-clock-prod: prod
-  # hi-worker-prod: prod
-  # hi-web-stag: stag
-  # hi-clock-stag: stag
-  # hi-worker-stag: stag
-user: ec2-user
+bastion:
+  host_key_check: false
+  # host:
+
+ssh:
+  user: ec2-user
+
+ecs_service_cluster_map:
+  default: default

data/lib/sonic/docker.rb

@@ -19,6 +19,7 @@ module Sonic
 
     def setup
       validate!
+      confirm_ssh_access
       copy_over_container_data
     end
 
@@ -29,7 +30,7 @@ module Sonic
 
     # command is an Array
     def execute(*command)
-      UI.say "=> #{command.join(' ')}".colorize(:green)
+      UI.say "=> #{command.join(' ')}".color(:green)
       success = system(*command)
       unless success
         UI.error(command.join(' '))
@@ -46,10 +47,37 @@ module Sonic
       kernel_exec(*args)
     end
 
+    def build_host
+      host = @bastion ? bastion_host : ssh_host
+      host = "#{@user}@#{host}" unless host.include?('@')
+      host
+    end
+
+    def confirm_ssh_access
+      host = build_host
+      puts "Checking access to instance #{detector.instance_id}"
+
+      ssh = ["ssh", ssh_options, "-At", host, "uptime", "2>&1"]
+      command = ssh.join(' ')
+      puts "=> #{command}".color(:green)
+      output = `#{command}`
+      if output.include?("Permission denied")
+        puts output
+        UI.error("Access to the instance denied. Maybe check your ssh keys.")
+        exit 1
+      elsif output.include?(" up ")
+        puts "Access OK!"
+      else
+        puts output
+        UI.error("There was an error trying to access the instnace.")
+        exit 1
+      end
+    end
+
     def copy_over_container_data
       create_container_data
 
-      host = @bastion ? bastion_host : ssh_host
+      host = build_host
 
       # LEVEL 1
       # Always clean up remote /tmp/sonic in case of previous interrupted run.

data/lib/sonic/ecs.rb

@@ -0,0 +1,22 @@
+module Sonic
+  autoload :Docker, 'sonic/docker'
+
+  class Ecs < BaseCommand
+
+    class_option :bastion, desc: "Bastion jump host to use.  Defaults to no bastion server."
+    class_option :cluster, desc: "ECS Cluster to use.  Default cluster is default"
+
+    desc "exec [ECS_SERVICE]", "docker exec into running docker container associated with the service on a container instance"
+    long_desc Help.text("ecs/exec")
+    def exec(service, *command)
+      Docker.new(service, options.merge(command: command)).exec
+    end
+
+    # Cannot name the command run because that is a reserved Thor keyword :(
+    desc "sh [ECS_SERVICE]", "docker run with the service on a container instance"
+    long_desc Help.text("ecs/sh")
+    def sh(service, *command)
+      Docker.new(service, options.merge(command: command)).run
+    end
+  end
+end

data/lib/sonic/execute.rb

@@ -1,11 +1,15 @@
+require 'yaml'
+require 'active_support/core_ext/hash'
+
 module Sonic
   class Execute
-    include AwsServices
+    include AwsService
 
     def initialize(command, options)
       @command = command
       @options = options
-      @filter = @options[:filter].split(',').map{|s| s.strip}
+      @tags = @options[:tags]
+      @instance_ids = @options[:instance_ids]
     end
 
     # aws ssm send-command \
@@ -15,40 +19,193 @@ module Sonic
     #   --parameters '{"commands":["#!/usr/bin/python","print \"Hello world from python\""]}' \
     #   --query "Command.CommandId"
     def execute
+      check_filter_options!
       ssm_options = build_ssm_options
       if @options[:noop]
         UI.noop = true
-        command_id = "fake command id"
+        command_id = "fake command id for noop mode"
         success = true # fake it for specs
       else
         instances_count = check_instances
         return unless instances_count > 0
 
         success = nil
+        puts "Sending command to SSM with options:"
+        puts YAML.dump(ssm_options.deep_stringify_keys)
+        puts
         begin
-          resp = ssm.send_command(ssm_options)
+          resp = send_command(ssm_options)
+
           command_id = resp.command.command_id
           success = true
         rescue Aws::SSM::Errors::InvalidInstanceId => e
           ssm_invalid_instance_error_message(e)
         end
       end
-      if success
-        UI.say "Command sent to AWS SSM. To check the details of the command:"
-        display_list_command(command_id)
+
+      return unless success
+
+      # IF COMMAND IS ONLY ON A SINGLE INSTANCE THEN WILL DISPLAY A BUNCH OF
+      # INFO ON THE INSTANCE. IF ITS A LOT OF INSTANCES, THEN SHOW A SUMMARY
+      # OF COMMANDS THAT WILL LEAD TO THE OUTPUT OF EACH INSTANCE.
+      UI.say "Command sent to AWS SSM. To check the details of the command:"
+      display_ssm_commands(command_id, ssm_options)
+      puts
+      return if @options[:noop]
+      status = wait(command_id)
+      instances_found = display_ssm_output(command_id)
+      display_console_url(command_id)
+
+      if status == "Success"
+        puts "Command successful: #{status}".color(:green)
+        exit(0)
+      else
+        puts "Command unsuccessful: #{status}".color(:red)
+        exit(1)
+      end
+    end
+
+    def wait(command_id)
+      ongoing_states = ["Pending", "InProgress", "Delayed"]
+
+      print "Waiting for ssm command to finish..."
+      resp = ssm.list_commands(command_id: command_id)
+      status = resp["commands"].first["status"]
+      while ongoing_states.include?(status)
+        resp = ssm.list_commands(command_id: command_id)
+        status = resp["commands"].first["status"]
+        sleep 1
+        print '.'
+      end
+      puts "\nCommand finished."
+      puts
+      status
+    end
+
+    def display_ssm_output(command_id)
+      resp = ssm.list_command_invocations(command_id: command_id)
+      command_invocations = resp.command_invocations
+      command_invocation = command_invocations.first
+      unless command_invocation
+        puts "WARN: No instances found that matches the --tags or --instance-ids option".color(:yellow)
+        return false # instances_found
+      end
+      instance_id = command_invocation.instance_id
+
+      if command_invocations.size > 1
+        puts "Multiple instance targets. Total targets: #{command_invocations.size}. Only displaying output for #{instance_id}."
+      else
+        puts "Displaying output for #{instance_id}."
+      end
+
+      resp = ssm.get_command_invocation(
+        command_id: command_id, instance_id: instance_id
+      )
+      puts "Command status: #{colorized_status(resp["status"])}"
+      ssm_output(resp, "output")
+      ssm_output(resp, "error")
+      puts
+      true # instances_found
+    end
+
+    def display_console_url(command_id)
+      region = `aws configure get region`.strip rescue 'us-east-1'
+      console_url = "https://#{region}.console.aws.amazon.com/systems-manager/run-command/#{command_id}"
+      puts "To see the more output details visit:"
+      puts "  #{console_url}"
+      puts
+      copy_paste_clipboard(console_url, "Pro tip: the console url is already in your copy/paste clipboard.")
+    end
+
+    def colorized_status(status)
+      case status
+      when "Success"
+        status.color(:green)
+      when "Failed"
+        status.color(:red)
+      else
+        status
       end
     end
 
+    # type: output or error
+    def ssm_output(resp, type)
+      content_key = "standard_#{type}_content"
+      s3_key = "standard_#{type}_url"
+
+      content = resp[content_key]
+      return if content.empty?
+
+      puts "Command standard #{type}:"
+      # "https://s3.amazonaws.com/infra-prod/ssm/commands/sonic/0a4f4bef-8f63-4235-8b30-ae296477261a/i-0b2e6e187a3f9ada9/awsrunPowerShellScript/0.awsrunPowerShellScript/stderr">
+      if content.include?("--output truncated--") && !resp[s3_key].empty?
+        s3_url = resp[s3_key]
+        info = s3_url.sub('https://s3.amazonaws.com/', '').split('/')
+        bucket = info[0]
+        key = info[1..-1].join('/')
+        resp = s3.get_object(bucket: bucket, key: key)
+        data = resp.body.read
+        puts data
+
+        path = "/tmp/sonic-output.txt"
+        puts "------"
+        puts "Output also written to #{path}"
+        IO.write(path, data)
+      else
+        puts content
+      end
+
+      # puts "#{s3_key}: #{resp[s3_key]}"
+    end
+
+    def send_command(options)
+      retries = 0
+
+      begin
+        resp = ssm.send_command(options)
+      rescue Aws::SSM::Errors::UnsupportedPlatformType
+        retries += 1
+        # toggle AWS-RunShellScript / AWS-RunPowerShellScript
+        options[:document_name] =
+          options[:document_name] == "AWS-RunShellScript" ?
+          "AWS-RunPowerShellScript" : "AWS-RunShellScript"
+
+        puts "#{$!}"
+        puts "Retrying with document_name #{options[:document_name]}"
+        puts "Retries: #{retries}"
+
+        retries <= 1 ? retry : raise
+      end
+
+      resp
+    end
+
     def build_ssm_options
-      criteria = transform_filter(@filter)
+      criteria = transform_filter_option
       command = build_command(@command)
-      criteria.merge(
-        document_name: "AWS-RunShellScript",
-        comment: "sonic #{ARGV.join(' ')}",
-        parameters: {
-          "commands" => command
-        }
+      options = criteria.merge(
+        document_name: "AWS-RunShellScript", # default
+        comment: "sonic #{ARGV.join(' ')}"[0..99], # comment has a max of 100 chars
+        parameters: { "commands" => command },
+        # Default CloudWatchLog settings. Can be overwritten with settings.yml send_command
+        # IMPORTANT: make sure the EC2 instance the command runs on has access to write to CloudWatch Logs.
+        cloud_watch_output_config: {
+          # cloud_watch_log_group_name: "ssm", # Defaults to /aws/ssm/AWS-RunShellScript (aws/ssm/SystemsManagerDocumentName https://amzn.to/38TKVse)
+          cloud_watch_output_enabled: true,
+        },
       )
+      settings_options = settings["send_command"] || {}
+      options.merge(settings_options.deep_symbolize_keys)
+    end
+
+    def settings
+      @settings ||= Setting.new.data
+    end
+
+    def check_filter_options!
+      return if @tags || @instance_ids
+      puts "ERROR: Please provide --tags or --instance-ids option".color(:red)
+      exit 1
     end
 
     #
@@ -58,41 +215,31 @@ module Sonic
     #
     # Examples
     #
-    #   transform_filter(["hi-web-prod", "hi-worker-prod", "i-006a097bb10643e20"])
+    #   transform_filter_option
     #   # => {
     #      instance_ids: ["i-006a097bb10643e20"],
     #      targets: [{key: "Name", values: "hi-web-prod,hi-worker-prod"}]
     #     }
     #
     # Returns the duplicated String.
-    def transform_filter(filter)
-      valid = validate_filter(filter)
-      unless valid
-        UI.error("The filter you provided '#{filter.join(',')}' is not valid.")
-        UI.say("The filter must either be all instance ids or just a list of tag names.")
-        exit 1
-      end
-
-      if filter.detect { |i| instance_id?(i) }
-        instance_ids = filter
-        {instance_ids: instance_ids}
-      else
-        tags = filter
-        targets = [{
-          key: "tag:#{tag_name}",
-          values: tags
-        }]
+    def transform_filter_option
+      if @tags
+        list = @tags.split(';')
+        targets = list.inject([]) do |final,item|
+          tag_name,value_list = item.split('=')
+          values = value_list.split(',').map(&:strip)
+          # structure expected by ssm send_command
+          option = {
+            key: "tag:#{tag_name}",
+            values: values
+          }
+          final << option
+          final
+        end
         {targets: targets}
-      end
-    end
-
-    # Either all instance ids are no instance ids is a valid filter
-    def validate_filter(filter)
-      if filter.detect { |i| instance_id?(i) }
-        instance_ids = filter.select { |i| instance_id?(i) }
-        instance_ids.size == filter.size
-      else
-        true
+      else # @instance_ids
+        instance_ids = @instance_ids.split(',')
+        {instance_ids: instance_ids}
       end
     end
 
@@ -109,7 +256,7 @@ module Sonic
         # The script is being feed inline so just join the command together into one script.
         # Still keep in an array form because that's how ssn.send_command works with AWS-RunShellScript
         # usually reads the command.
-        [command.join(" ")]
+        command.is_a?(Array) ? command : [command]
       end
     end
 
@@ -124,7 +271,7 @@ You can use the following command to check registered instances to SSM.
 #{ssm_describe_command}
       EOS
       UI.warn(message)
-      copy_paste_clipboard(ssm_describe_command)
+      copy_paste_clipboard(ssm_describe_command, "Pro tip: ssm describe-instance-information already in your copy/paste clipboard.")
     end
 
     def file_path?(command)
@@ -136,7 +283,7 @@ You can use the following command to check registered instances to SSM.
     def file_path(command)
       path = command.first
       path = path.sub('file://', '')
-      path = "#{@options[:project_root]}/#{path}" if @options[:project_root]
+      path = "#{Sonic.root}/#{path}"
       path
     end
 
@@ -150,7 +297,7 @@ You can use the following command to check registered instances to SSM.
       instances = List.new(@options).instances
       if instances.count == 0
         message = <<-EOL
-  Unable to find any instances with filter #{@filter.join(',')}.
+Unable to find any instances with filter #{@filter.join(',')}.
   Are you sure you specify the filter with either a EC2 tag or list instance ids?
   If you are using ECS identifiers, they are not supported with this command.
 EOL
@@ -170,16 +317,21 @@ EOL
       text =~ /i-.{17}/ || text =~ /i-.{8}/
     end
 
-    def display_list_command(command_id)
-      list_command = "aws ssm list-commands --command-id #{command_id}"
+    def display_ssm_commands(command_id, ssm_options)
+      list_command = "  aws ssm list-commands --command-id #{command_id}"
       UI.say list_command
-      copy_paste_clipboard(list_command)
+
+      return unless ssm_options[:instance_ids]
+      ssm_options[:instance_ids].each do |instance_id|
+        get_command = "  aws ssm get-command-invocation --command-id #{command_id} --instance-id #{instance_id}"
+        UI.say get_command
+      end
     end
 
-    def copy_paste_clipboard(command)
+    def copy_paste_clipboard(command, text)
       return unless RUBY_PLATFORM =~ /darwin/
       system("echo '#{command}' | pbcopy")
-      UI.say "Pro tip: the aws ssm command is already in your copy/paste clipboard."
+      UI.say text
     end
   end
 end