sonic-screwdriver 1.4.0 → 2.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. checksums.yaml +4 -4
  2. data/.circleci/bin/commit_docs.sh +26 -0
  3. data/.circleci/config.yml +72 -0
  4. data/.gitignore +2 -1
  5. data/CHANGELOG.md +29 -3
  6. data/Gemfile +3 -3
  7. data/Guardfile +17 -10
  8. data/LICENSE.txt +2 -2
  9. data/README.md +25 -28
  10. data/Rakefile +9 -2
  11. data/docs/_config.yml +3 -0
  12. data/docs/_docs/help.md +1 -1
  13. data/docs/_docs/install-bastion.md +5 -15
  14. data/docs/_docs/install.md +3 -13
  15. data/docs/_docs/next-steps.md +1 -1
  16. data/docs/_docs/settings.md +42 -56
  17. data/docs/_docs/tutorial-ecs-exec.md +16 -20
  18. data/docs/_docs/tutorial-ecs-sh.md +73 -0
  19. data/docs/_docs/tutorial-execute.md +106 -38
  20. data/docs/_docs/tutorial-ssh.md +15 -19
  21. data/docs/_docs/why-ec2-run-command.md +1 -1
  22. data/docs/_includes/commands.html +5 -5
  23. data/docs/_includes/content.html +5 -0
  24. data/docs/_includes/css/main.css +15 -9
  25. data/docs/_includes/css/sonic.css +7 -5
  26. data/docs/_includes/example.html +4 -4
  27. data/docs/_includes/footer.html +6 -4
  28. data/docs/_includes/reference.md +1 -0
  29. data/docs/_includes/subnav.html +2 -1
  30. data/docs/_reference/sonic-completion.md +44 -0
  31. data/docs/_reference/sonic-completion_script.md +25 -0
  32. data/docs/_reference/sonic-ecs-exec.md +30 -0
  33. data/docs/_reference/sonic-ecs-help.md +21 -0
  34. data/docs/_reference/sonic-ecs-sh.md +35 -0
  35. data/docs/_reference/sonic-ecs.md +25 -0
  36. data/docs/_reference/sonic-execute.md +85 -0
  37. data/docs/_reference/sonic-list.md +40 -0
  38. data/docs/_reference/sonic-ssh.md +86 -0
  39. data/docs/_reference/sonic-version.md +21 -0
  40. data/docs/bin/web +1 -1
  41. data/docs/img/tutorials/ec2-console-run-command.png +0 -0
  42. data/docs/quick-start.md +17 -22
  43. data/docs/reference.md +12 -0
  44. data/{bin → exe}/sonic +3 -3
  45. data/lib/bash_scripts/docker-exec.sh +1 -0
  46. data/lib/bash_scripts/docker-run.sh +8 -1
  47. data/lib/sonic.rb +11 -3
  48. data/lib/sonic/{aws_services.rb → aws_service.rb} +6 -1
  49. data/lib/sonic/base_command.rb +82 -0
  50. data/lib/sonic/checks.rb +2 -2
  51. data/lib/sonic/cli.rb +41 -29
  52. data/lib/sonic/command.rb +8 -22
  53. data/lib/sonic/completer.rb +161 -0
  54. data/lib/sonic/completer/script.rb +6 -0
  55. data/lib/sonic/completer/script.sh +10 -0
  56. data/lib/sonic/core.rb +15 -0
  57. data/lib/sonic/default/settings.yml +9 -16
  58. data/lib/sonic/docker.rb +30 -2
  59. data/lib/sonic/ecs.rb +22 -0
  60. data/lib/sonic/execute.rb +203 -51
  61. data/lib/sonic/help.rb +9 -0
  62. data/lib/sonic/help/command/send.md +10 -0
  63. data/lib/sonic/help/completion.md +22 -0
  64. data/lib/sonic/help/completion_script.md +3 -0
  65. data/lib/sonic/help/ecs/exec.md +8 -0
  66. data/lib/sonic/help/ecs/sh.md +13 -0
  67. data/lib/sonic/help/execute.md +59 -0
  68. data/lib/sonic/help/list.md +17 -0
  69. data/lib/sonic/help/ssh.md +60 -0
  70. data/lib/sonic/list.rb +5 -2
  71. data/lib/sonic/setting.rb +47 -0
  72. data/lib/sonic/ssh.rb +42 -23
  73. data/lib/sonic/ssh/identifier_detector.rb +7 -3
  74. data/lib/sonic/ui.rb +2 -2
  75. data/lib/sonic/version.rb +1 -1
  76. data/sonic.gemspec +14 -9
  77. data/spec/lib/cli_spec.rb +11 -11
  78. data/spec/lib/sonic/execute_spec.rb +1 -2
  79. data/spec/spec_helper.rb +18 -10
  80. metadata +115 -19
  81. data/Gemfile.lock +0 -134
  82. data/docs/_docs/tutorial-ecs-run.md +0 -100
  83. data/lib/sonic/cli/help.rb +0 -152
  84. data/lib/sonic/settings.rb +0 -115
data/lib/sonic/help.rb ADDED
@@ -0,0 +1,9 @@
1
+ module Sonic::Help
2
+ class << self
3
+ def text(namespaced_command)
4
+ path = namespaced_command.to_s.gsub(':','/')
5
+ path = File.expand_path("../help/#{path}.md", __FILE__)
6
+ IO.read(path) if File.exist?(path)
7
+ end
8
+ end
9
+ end
@@ -0,0 +1,10 @@
1
+ Run as a command across a list of servers. A filter must be provided. The filter can be a mix of instance ids and ec2 tags. This command can also take a path to a file. To specify a path to a file use file:// at the beginning of your file.
2
+
3
+ Examples:
4
+
5
+ $ sonic command send hi-web-prod uptime
6
+ $ sonic command send hi-web-prod,hi-worker-prod,hi-clock-prod uptime
7
+ $ sonic command send i-030033c20c54bf149,i-030033c20c54bf150 uname -a
8
+ $ sonic command send i-030033c20c54bf149 file://hello.sh
9
+
10
+ You cannot mix instance ids and tag names in the filter.
@@ -0,0 +1,22 @@
1
+ Example:
2
+
3
+ sonic completion
4
+
5
+ Prints words for TAB auto-completion.
6
+
7
+ Examples:
8
+
9
+ sonic completion
10
+ sonic completion execute
11
+ sonic completion list
12
+
13
+ To enable, TAB auto-completion add the following to your profile:
14
+
15
+ eval $(sonic completion_script)
16
+
17
+ Auto-completion example usage:
18
+
19
+ sonic [TAB]
20
+ sonic exe[TAB]
21
+ sonic execute [TAB]
22
+ sonic list [TAB]
@@ -0,0 +1,3 @@
1
+ To use, add the following to your `~/.bashrc` or `~/.profile`
2
+
3
+ eval $(sonic completion_script)
@@ -0,0 +1,8 @@
1
+ Ssh into an ECS container instance, finds a running docker container associated
2
+ with the service and docker exec's into it.
3
+
4
+ Examples:
5
+
6
+ $ sonic ecs exec my-service --cluster my-cluster
7
+
8
+ You can use a variety of identifiers. These include the ECS service name and task id.
@@ -0,0 +1,13 @@
1
+ Ssh into an ECS container instance and runs a docker container using the same
2
+ environment and image as the specified running service.
3
+
4
+ Examples:
5
+
6
+ $ sonic ecs sh --cluster my-cluster my-service
7
+ $ sonic ecs sh --cluster my-cluster my-service
8
+
9
+ # If there are flags in the command that you want to pass down to the docker
10
+ run command you will need to put the command in single quotes. This is due to
11
+ the way Thor (what this tool uses) parses options.
12
+
13
+ $ sonic ecs sh --cluster production-hi hi-web 'rake -T'
@@ -0,0 +1,59 @@
1
+ * A filter must be provided. The filter can be a mix of instance ids and ec2 tags.
2
+ * The command can be provided inline or as a file. To a file use `file://` at the beginning of your file.
3
+
4
+ ## Examples Summary
5
+
6
+ sonic execute --tags Name=demo-web uptime
7
+ sonic execute --tags Name=demo-web,demo-worker uptime # multiple values
8
+ sonic execute --instance-ids i-030033c20c54bf149,i-030033c20c54bf150 uname -a
9
+ sonic execute --instance-ids i-030033c20c54bf149 file://hello.sh # script from file
10
+
11
+ You cannot mix instance ids and tag names in the filter.
12
+
13
+ ## Example Detailed
14
+
15
+ Here's a command example output in detailed:
16
+
17
+ $ sonic execute --instance-ids i-0bf51a000ab4e73a8 uptime
18
+ Sending command to SSM with options:
19
+ ---
20
+ instance_ids:
21
+ - i-0bf51a000ab4e73a8
22
+ document_name: AWS-RunShellScript
23
+ comment: sonic execute --instance-ids i-0bf51a000ab4e73a8 uptime
24
+ parameters:
25
+ commands:
26
+ - uptime
27
+ output_s3_region: us-east-1
28
+ output_s3_bucket_name: [reacted]
29
+ output_s3_key_prefix: ssm/commands/sonic
30
+
31
+ Command sent to AWS SSM. To check the details of the command:
32
+ aws ssm list-commands --command-id 0bb18d58-6436-49fd-9bfd-0c4b6c51c7a2
33
+ aws ssm get-command-invocation --command-id 0bb18d58-6436-49fd-9bfd-0c4b6c51c7a2 --instance-id i-0bf51a000ab4e73a8
34
+
35
+ Waiting for ssm command to finish.....
36
+ Command finished.
37
+
38
+ Displaying output for i-0bf51a000ab4e73a8.
39
+ Command status: Success
40
+ Command standard output:
41
+ 01:08:10 up 8 days, 6:41, 0 users, load average: 0.00, 0.00, 0.00
42
+
43
+ To see the more output details visit:
44
+ https://us-west-2.console.aws.amazon.com/systems-manager/run-command/0bb18d58-6436-49fd-9bfd-0c4b6c51c7a2
45
+
46
+ Pro tip: the console url is already in your copy/paste clipboard.
47
+ $
48
+
49
+ Notice the conveniences of `sonic execute`, it:
50
+
51
+ 1. Showed the parameters that will be sent as part of the send_command call to SSM.
52
+ 2. Sent the command to SSM.
53
+ 3. Waited for the command to finish.
54
+ 4. Displayed the output of the command.
55
+ 5. Provided the console url that visit to view more details about the SSM command.
56
+
57
+ The AWS SSM console looks like this:
58
+
59
+ <img src="/img/tutorials/ec2-console-run-command.png" class="doc-photo" />
@@ -0,0 +1,17 @@
1
+ A filter must be provided. The filter can be a mix of instance ids and ec2 tags. sonic list will auto-detect the what type of filter it is. The filter is optional.
2
+
3
+ ## Examples
4
+
5
+ $ sonic list
6
+ $ sonic list hi-web-prod
7
+ $ sonic list hi-web-prod,hi-clock-prod
8
+ $ sonic list i-09482b1a6e330fbf7
9
+
10
+ ## Example Output
11
+
12
+ $ sonic list --header i-09482b1a6e330fbf7
13
+ Instance Id Public IP Private IP Type
14
+ i-09482b1a6e330fbf7 54.202.152.168 172.31.21.108 t2.small
15
+ $
16
+
17
+ You cannot mix instance ids and tag names in the filter.
@@ -0,0 +1,60 @@
1
+ * EC2 instance id. Example: i-067c5e3f026c1e801
2
+ * EC2 tag value. Any tag value is search, the tag key does not matter only the tag value matters. Example: hi-web
3
+ * ECS service. Example: my-ecs-service
4
+ * ECS container instance id. Example: 7fbc8c75-4675-4d39-a5a4-0395ff8cd474
5
+ * ECS task id. Example: 1ed12abd-645c-4a05-9acf-739b9d790170
6
+
7
+ When using ecs identifiers the `--cluster` option is required or can be set in ~/.sonic/settings.yml.
8
+
9
+ Examples:
10
+
11
+ $ sonic ssh i-067c5e3f026c1e801
12
+ $ sonic ssh hi-web
13
+ $ sonic ssh --cluster my-cluster my-ecs-service
14
+ $ sonic ssh 7fbc8c75-4675-4d39-a5a4-0395ff8cd474
15
+ $ sonic ssh 1ed12abd-645c-4a05-9acf-739b9d790170
16
+
17
+ Sonic ssh builds up the ssh command and shells out to it. For example, the following commands:
18
+
19
+ sonic ssh i-027363802c6ff314f
20
+
21
+ Translates to:
22
+
23
+ ssh ec2-user@ec2-52-24-216-170.us-west-2.compute.amazonaws.com
24
+
25
+ You can also tack on any command to be run at the end of the command. Example:
26
+
27
+ $ sonic ssh i-027363802c6ff314f uptime
28
+ => ssh ec2-user@ec2-52-24-216-170.us-west-2.compute.amazonaws.com uptime
29
+ 15:57:02 up 18:21, 0 users, load average: 0.00, 0.01, 0.00
30
+
31
+ ## Specifying pem keys
32
+
33
+ The recommended way to specify custom private keys is to use ssh-agent as covered here: https://blog.boltops.com/2017/09/21/3-ssh-tips-ssh-agent-tunnel-and-escaping-from-the-dead
34
+
35
+ But you can also specify the pem key to use with the -i option. Example:
36
+
37
+ $ sonic ssh -i ~/.ssh/id_rsa-custom ec2-user@ec2-52-24-216-170.us-west-2.compute.amazonaws.com
38
+
39
+ ## Retry option
40
+
41
+ For newly launched instances, the instance's ssh access might not be quite ready. Typically, you must press up enter repeatedly until the instance is ready. Sonic ssh has a retry option that automates this. Example:
42
+
43
+ $ sonic ssh -r i-027363802c6ff314f
44
+
45
+ Bastion Host Support
46
+
47
+ Sonic ssh also supports a bastion host.
48
+
49
+ $ sonic ssh --bastion bastion.domain.com i-027363802c6ff314f
50
+ $ sonic ssh --bastion user@bastion.domain.com i-027363802c6ff314f
51
+
52
+ Here's what the output looks like:
53
+
54
+ $ sonic ssh --bastion 34.211.223.3 i-0f7f833131a51ce35 uptime
55
+ => ssh -At ec2-user@34.211.223.3 ssh ec2-user@10.10.110.135 uptime
56
+ 17:57:59 up 37 min, 0 users, load average: 0.00, 0.02, 0.00
57
+ Connection to 34.211.223.3 closed.
58
+ $
59
+
60
+ You can also set the bastion host and other options with a [settings file](http://sonic-screwdriver.cloud/docs/settings/).
data/lib/sonic/list.rb CHANGED
@@ -1,6 +1,6 @@
1
1
  module Sonic
2
2
  class List
3
- include AwsServices
3
+ include AwsService
4
4
 
5
5
  def initialize(options)
6
6
  @options = options
@@ -22,6 +22,9 @@ module Sonic
22
22
  # ERROR: The instance ID 'i-066b140d9479e9682' does not exist
23
23
  UI.error(e.message)
24
24
  exit 1
25
+ rescue Aws::EC2::Errors::InvalidInstanceIDMalformed => e
26
+ UI.error(e.message)
27
+ exit 1
25
28
  end
26
29
  instances
27
30
  end
@@ -32,7 +35,7 @@ module Sonic
32
35
 
33
36
  if @options[:header] && !zero_instances
34
37
  header = ["Instance Id", "Name", "Public IP", "Private IP", "Type"]
35
- UI.say header.join("\t").colorize(:green)
38
+ UI.say header.join("\t").color(:green)
36
39
  end
37
40
 
38
41
  instances.each do |i|
@@ -0,0 +1,47 @@
1
+ require 'yaml'
2
+ require 'memoist'
3
+ require 'active_support/core_ext/hash'
4
+
5
+ module Sonic
6
+ class Setting
7
+ extend Memoist
8
+
9
+ def data
10
+ settings_file = Sonic.profile || 'default'
11
+ settings_file += ".yml"
12
+
13
+ project = yaml_file("#{Sonic.root}/.sonic/#{settings_file}")
14
+ user = yaml_file("#{home}/.sonic/#{settings_file}")
15
+ default_file = File.expand_path("../default/settings.yml", __FILE__)
16
+ default = yaml_file(default_file)
17
+
18
+ data = merge(default, user, project)
19
+
20
+ if ENV['DEBUG_SETTINGS']
21
+ puts "settings data:"
22
+ pp data
23
+ end
24
+ data
25
+ end
26
+ memoize :data
27
+
28
+ def merge(*hashes)
29
+ hashes.inject({}) do |result, hash|
30
+ # note: important to compact for keys with nil value
31
+ result.deep_merge(hash.compact)
32
+ end
33
+ end
34
+
35
+ # Any empty file will result in "false". Lets ensure that an empty file
36
+ # loads an empty hash instead.
37
+ def yaml_file(path)
38
+ return {} unless File.exist?(path)
39
+ YAML.load_file(path) || {}
40
+ end
41
+
42
+ def home
43
+ # hack but fast
44
+ ENV['TEST'] ? "spec/fixtures/home" : ENV['HOME']
45
+ end
46
+ end
47
+ end
data/lib/sonic/ssh.rb CHANGED
@@ -1,11 +1,9 @@
1
- require 'colorize'
2
-
3
1
  module Sonic
4
2
  class Ssh
5
3
  autoload :IdentifierDetector, 'sonic/ssh/identifier_detector'
6
4
  autoload :CliOptions, 'sonic/ssh/cli_options'
7
5
 
8
- include AwsServices
6
+ include AwsService
9
7
  include CliOptions
10
8
 
11
9
  def initialize(identifier, options)
@@ -14,11 +12,12 @@ module Sonic
14
12
  @user, @identifier = extract_user!(identifier) # extracts/strips user from identifier
15
13
  # While --user option is supported at the class level, don't expose at the CLI level
16
14
  # to encourage users to use user@host notation.
17
- @user ||= options[:user] || settings.data["user"]
15
+ @user ||= options[:user] || settings["ssh"]["user"]
18
16
 
19
17
  @service = @identifier # always set service even though it's not always used as the identifier
20
- @cluster = options[:cluster] || settings.default_cluster(@service)
21
- @bastion = options[:bastion] || settings.default_bastion(@bastion)
18
+ map = settings["ecs_service_cluster_map"]
19
+ @cluster = options[:cluster] || map[@service] || map["default"] || "default"
20
+ @bastion = options[:bastion] || settings["bastion"]["host"]
22
21
  end
23
22
 
24
23
  def run
@@ -47,53 +46,71 @@ module Sonic
47
46
  def build_ssh_host
48
47
  return @identifier if ENV['TEST']
49
48
 
50
- detector = Ssh::IdentifierDetector.new(@cluster, @service, @identifier, @options)
51
49
  instance_id = detector.detect!
52
50
  instance_hostname(instance_id)
53
51
  end
54
52
 
53
+ def detector
54
+ @detector ||= Ssh::IdentifierDetector.new(@cluster, @service, @identifier, @options)
55
+ end
56
+
57
+
55
58
  def instance_hostname(ec2_instance_id)
56
59
  begin
57
60
  resp = ec2.describe_instances(instance_ids: [ec2_instance_id])
58
61
  rescue Aws::EC2::Errors::InvalidInstanceIDNotFound => e
59
62
  # e.message: The instance ID 'i-027363802c6ff3141' does not exist
60
- UI.error(e.message)
63
+ UI.error e.message
64
+ exit 1
65
+ rescue Aws::Errors::NoSuchEndpointError, SocketError
66
+ UI.error "It doesnt look like you have an internet connection. Please double check that you have an internet connection."
61
67
  exit 1
62
68
  end
63
69
  instance = resp.reservations[0].instances[0]
64
70
  # struct Aws::EC2::Types::Instance
65
71
  # http://docs.aws.amazon.com/sdkforruby/api/Aws/EC2/Types/Instance.html
66
- host = if @bastion
67
- instance.private_ip_address
68
- else
69
- instance.public_ip_address
70
- end
71
- "#{@user}@#{host}"
72
+ if @bastion
73
+ instance.private_ip_address
74
+ else
75
+ instance.public_ip_address
76
+ end
72
77
  end
73
78
 
74
79
  # Will use Kernel.exec so that the ssh process takes over this ruby process.
75
80
  def kernel_exec(*args)
76
81
  # append the optional command that can be provided to the ssh command
77
82
  full_command = args + @options[:command]
78
- puts "=> #{full_command.join(' ')}".colorize(:green)
83
+ puts "=> #{full_command.join(' ')}".color(:green)
79
84
  # https://ruby-doc.org/core-2.3.1/Kernel.html#method-i-exec
80
85
  # Using 2nd form
81
86
  Kernel.exec(*full_command) unless @options[:noop]
82
87
  end
83
88
 
84
89
  private
90
+ # direct access to settings data
85
91
  def settings
86
- @settings ||= Settings.new(@options[:project_root])
92
+ @settings ||= Setting.new.data
87
93
  end
88
94
 
89
95
  # Returns Array of flags.
90
96
  # Example:
91
97
  # ["-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null"]
92
98
  def ssh_options
93
- host_key_check_options = settings.host_key_check_options
94
99
  keys_option + host_key_check_options
95
100
  end
96
101
 
102
+ # By default bypass strict host key checking for convenience.
103
+ # But user can overrride this.
104
+ def host_key_check_options
105
+ if settings["bastion"]["host_key_check"] == true
106
+ []
107
+ else
108
+ # settings["bastion"]["host_key_check"] nil will disable checking also
109
+ # disables host key checking
110
+ %w[-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null]
111
+ end
112
+ end
113
+
97
114
  # Will prepend the bastion host if required
98
115
  # When bastion set
99
116
  # ssh [options] -At [bastion_host] ssh -At [ssh_host]
@@ -111,12 +128,14 @@ private
111
128
  # -A = Enables forwarding of the authentication agent connection
112
129
  # -t = Force pseudo-terminal allocati
113
130
  def build_ssh_command
114
- ssh = ["ssh"] + ssh_options
115
- ssh += ["-At", bastion_host, "ssh"] + ssh_options if @bastion
116
- # ssh_host is internal ip when bastion is set
117
- # ssh_host is public ip when bastion is not set
118
- ssh += ["-At", ssh_host]
119
- ssh
131
+ command = ["ssh", "-t"] + ssh_options
132
+ if @bastion
133
+ # https://en.wikibooks.org/wiki/OpenSSH/Cookbook/Proxies_and_Jump_Hosts
134
+ # -J xxx is -o ProxyJump=xxx
135
+ proxy_jump = ["-J", bastion_host]
136
+ command += proxy_jump
137
+ end
138
+ command += ["-t", "#{@user}@#{ssh_host}"]
120
139
  end
121
140
 
122
141
  # Private: Extracts and strips the user from the identifier.
@@ -4,7 +4,7 @@ class Ssh
4
4
  class IdentifierDetector
5
5
 
6
6
  include Ec2Tag
7
- include AwsServices
7
+ include AwsService
8
8
  include Checks
9
9
 
10
10
  def initialize(cluster, service, identifier, options)
@@ -17,7 +17,7 @@ class Ssh
17
17
  # Returns exactly 1 instance_id or exits the program.
18
18
  # The bang check_* methods can exit early.
19
19
  def detect!
20
- instance_id = case detected_type
20
+ @instance_id ||= case detected_type
21
21
  when :ecs_container_instance_or_task_arn
22
22
  check_cluster_exists! unless @options[:noop]
23
23
 
@@ -36,6 +36,10 @@ class Ssh
36
36
  end
37
37
  end
38
38
 
39
+ def instance_id
40
+ detect!
41
+ end
42
+
39
43
  # Any of these methods either returns exactly 1 instance_id or exit the program.
40
44
  #
41
45
  # Returns detected_type
@@ -69,7 +73,7 @@ class Ssh
69
73
 
70
74
  task = response.tasks.first
71
75
  unless task
72
- puts "Unable to find a #{task_arn.green} container instance or task in the #{@cluster.green} cluster."
76
+ puts "Unable to find a #{task_arn.color(:green)} container instance or task in the #{@cluster.color(:green)} cluster."
73
77
  exit 1
74
78
  end
75
79