sshkit 1.7.1 → 1.8.0

data/Gemfile

@@ -6,3 +6,10 @@ platforms :rbx do
   gem 'rubysl', '~> 2.0'
   gem 'json'
 end
+
+# Chandler requires Ruby >= 2.1.0, but depending on the Travis environment,
+# we may not meet that requirement. Only include the chandler gem if the Ruby
+# requirement is met. (Chandler is used only for `rake release`; see Rakefile.)
+if Gem::Requirement.new('>= 2.1.0').satisfied_by?(Gem::Version.new(RUBY_VERSION))
+  gem 'chandler', '>= 0.1.1'
+end

data/README.md

@@ -11,23 +11,82 @@ more servers.
 The typical use-case looks something like this:
 
 ```ruby
+require 'sshkit'
 require 'sshkit/dsl'
 
-on %w{1.example.com 2.example.com}, in: :sequence, wait: 5 do
+on %w{1.example.com 2.example.com}, in: :sequence, wait: 5 do |host|
   within "/opt/sites/example.com" do
     as :deploy  do
       with rails_env: :production do
         rake   "assets:precompile"
         runner "S3::Sync.notify"
-        execute "node", "socket_server.js"
+        execute :node, "socket_server.js"
       end
     end
   end
 end
 ```
 
-One will notice that it's quite low level, but exposes a convenient API, the
-`as()`/`within()`/`with()` are nestable in any order, repeatable, and stackable.
+You can find many other examples of how to use SSHKit over in [EXAMPLES.md](EXAMPLES.md).
+
+## Basic usage
+
+The `on()` method is used to specify the backends on which you'd like to run the commands.
+You can pass one or more hosts as parameters; this runs commands via SSH. Alternatively you can
+pass `:local` to run commands locally. By default SSKit will run the commands on all hosts in
+parallel.
+
+#### Running commands
+
+All backends support the `execute(*args)`, `test(*args)` & `capture(*args)` methods
+for executing a command. You can call any of these methods in the context of an `on()`
+block.
+
+**Note: In SSHKit, the first parameter of the `execute` / `test` / `capture` methods
+has a special significance. If the first parameter isn't a Symbol,
+SSHKit assumes that you want to execute the raw command and the
+`as` / `within` / `with` methods, `SSHKit.config.umask` and [the comand map](#the-command-map)
+have no effect.**
+
+Typically, you would pass a Symbol for the command name and it's args as follows:
+
+```ruby
+on '1.example.com'
+  if test("[ -f somefile.txt ]")
+    execute(:cp, 'somefile.txt', 'somewhere_else.txt')
+  end
+  ls_output = capture(:ls, '-l')
+end
+```
+
+By default the `capture` methods strips whitespace. If you need to preserve whitespace
+you can pass the `strip: false` option: `capture(:ls, '-l', strip: false)`
+
+#### Transferring files
+
+All backends also support the `upload!` and `download!` methods for transferring files.
+For the remote backed, the file is tranferred with scp.
+
+```ruby
+on '1.example.com' do
+  upload! 'some_local_file.txt', '/home/some_user/somewhere'
+  download! '/home/some_user/some_remote_file.txt', 'somewhere_local'
+end
+```
+
+#### Users, working directories, environment variables and umask
+
+When running commands, you can tell SSHKit to set up the context for those
+commands using the following methods:
+
+```ruby
+as(user: 'un', group: 'grp') { execute('cmd') } # Executes sudo -u un -- sh -c 'sg grp cmd'
+within('/somedir') { execute('cmd') }           # Executes cd /somedir && cmd
+with(env_var: 'value') { execute('cmd') }       # Executes ENV_VAR=value cmd
+SSHKit.config.umask = '077'                     # All commands are executed with umask 077 && cmd
+```
+
+The `as()` / `within()` / `with()` are nestable in any order, repeatable, and stackable.
 
 When used inside a block in this way, `as()` and `within()` will guard
 the block they are given with a check.
@@ -99,7 +158,7 @@ end
 # they will all exist (all tar commands exited with a success status, or
 # that we will have raised an exception if one of them failed.
 on all_servers do |host|
-  in site_dir do
+  within site_dir do
     backup_filename = "backup-#{host.hostname}.tar.gz"
     target_filename = "backups/#{Time.now.utc.iso8601}/#{host.hostname}.tar.gz"
     puts capture(:s3cmd, 'put', backup_filename, target_filename)
@@ -197,6 +256,145 @@ desirable.
 *Note:* All keys should be symbolised, as the *Command* object will symbolize it's
 first argument before attempting to find it in the *command map*.
 
+## Interactive commands
+> (BETA) (Added in version #.##)
+
+By default, commands against remote servers are run in a *non-login, non-interactive* ssh session.
+This is by design, to try and isolate the environment and make sure that things work as expected,
+regardless of any changes that might happen on the server side. This means that,
+although the server may have prompted you, and be waiting for it,
+**you cannot send data to the server by typing into your terminal window**.
+Wherever possible, you should call commands in a way that doesn't require interaction
+(eg by specifying all options as command arguments).
+
+However in some cases, you may want to programmatically drive interaction with a command
+and this can be achieved by specifying an `:interaction_handler` option when you `execute`, `capture` or `test` a command.
+
+**It is not necessary, or desirable to enable `Netssh.config.pty` to use the `interaction_handler` option.
+Only enable `Netssh.config.pty` if the command you are calling won't work without a pty.**
+
+An `interaction_handler` is an object which responds to `on_data(command, stream_name, data, channel)`.
+The `interaction_handler`'s `on_data` method will be called each time `stdout` or `stderr` data is available from
+the server. Data can be sent back to the server using the `channel` parameter. This allows scripting of command
+interaction by responding to `stdout` or `stderr` lines with any input required.
+
+For example, an interaction handler to change the password of your linux user using the `passwd` command could look like this:
+
+```ruby
+class PasswdInteractionHandler
+  def on_data(command, stream_name, data, channel)
+    puts data
+    case data
+      when '(current) UNIX password: '
+        channel.send_data("old_pw\n")
+      when 'Enter new UNIX password: ', 'Retype new UNIX password: '
+        channel.send_data("new_pw\n")
+      when 'passwd: password updated successfully'
+      else
+        raise "Unexpected stderr #{stderr}"
+    end
+  end
+end
+
+# ...
+
+execute(:passwd, interaction_handler: PasswdInteractionHandler.new)
+```
+
+#### Using the `SSHKit::MappingInteractionHandler`
+
+Often, you want to map directly from a short output string returned by the server (either stdout or stderr)
+to a corresponding input string (as in the case above). For this case you can specify
+the `interaction_handler` option as a hash. This is used to create a `SSHKit::MappingInteractionHandler` which
+provides similar functionality to the linux [expect](http://expect.sourceforge.net/) library:
+
+```ruby
+execute(:passwd, interaction_handler: {
+  '(current) UNIX password: ' => "old_pw\n",
+  /(Enter|Retype) new UNIX password: / => "new_pw\n"
+})
+```
+
+Note: the key to the hash keys are matched against the server output `data` using the case equals `===` method.
+This means that regexes and any objects which define `===` can be used as hash keys.
+
+Hash keys are matched in order, which allows for default wildcard matches:
+
+```ruby
+execute(:my_command, interaction_handler: {
+  "some specific line\n" => "specific input\n",
+  /.*/ => "default input\n"
+})
+```
+
+You can also pass a Proc object to map the output line from the server:
+
+```ruby
+execute(:passwd, interaction_handler: lambda { |server_data|
+  case server_data
+  when '(current) UNIX password: '
+    "old_pw\n",
+  when /(Enter|Retype) new UNIX password: /
+    "new_pw\n"
+  end
+})
+```
+
+`MappingInteractionHandler`s are stateless, so you can assign one to a constant and reuse it:
+
+```ruby
+ENTER_PASSWORD = SSHKit::MappingInteractionHandler.new(
+  "Please Enter Password\n" => "some_password\n"
+)
+
+execute(:first_command, interaction_handler: ENTER_PASSWORD)
+execute(:second_command, interaction_handler: ENTER_PASSWORD)
+```
+
+#### Exploratory logging
+
+By default, the `MappingInteractionHandler` does not log, in case the server output or input contains sensitive
+information. However, if you pass a second `log_level` parameter to the constructor, the `MappingInteractionHandler`
+will log information about what output is being returned by the server, and what input is being sent
+in response. This can be helpful if you don't know exactly what the server is sending back (whitespace, newlines etc).
+
+```ruby
+  # Start with this and run your script
+  execute(:unfamiliar_command, MappingInteractionHandler.new({}, :debug))
+  # DEBUG log => Unable to find interaction handler mapping for stdout:
+  #              "Please type your input:\r\n" so no response was sent"
+
+  # Update mapping:
+  execute(:unfamiliar_command, MappingInteractionHandler.new(
+    {"Please type your input:\r\n" => "Some input\n"}
+    :debug
+  ))
+```
+
+#### The `data` parameter
+
+The `data` parameter of `on_data(command, stream_name, data, channel)` is a string containing the latest data
+delivered from the backend.
+
+When using the `Netssh` backend for commands where a small amount of data is returned (eg prompting for sudo passwords),
+`on_data` will normally be called once per line and `data` will be terminated by a newline. For commands with
+larger amounts of output, `data` is delivered as it arrives from the underlying network stack, which depends on
+network conditions, buffer sizes, etc. In this case, you may need to implement a more complex `interaction_handler`
+to concatenate `data` from multiple calls to `on_data` before matching the required output.
+
+When using the `Local` backend, `on_data` is always called once per line.
+
+#### The `channel` parameter
+
+When using the `Netssh` backend, the `channel` parameter of `on_data(command, stream_name, data, channel)` is a
+[Net::SSH Channel](http://net-ssh.github.io/ssh/v2/api/classes/Net/SSH/Connection/Channel.html).
+When using the `Local` backend, it is a [ruby IO](http://ruby-doc.org/core/IO.html) object.
+If you need to support both sorts of backends with the same interaction handler,
+you need to call methods on the appropriate API depending on the channel type.
+One approach is to detect the presence of the API methods you need -
+eg `channel.respond_to?(:send_data) # Net::SSH channel` and `channel.respond_to?(:write) # IO`.
+See the `SSHKit::MappingInteractionHandler` for an example of this.
+
 ## Output Handling
 
 ![Example Output](https://raw.github.com/leehambley/sshkit/master/examples/images/example_output.png)
@@ -204,15 +402,63 @@ first argument before attempting to find it in the *command map*.
 By default, the output format is set to `:pretty`:
 
 ```ruby
-SSHKit.config.format = :pretty
+SSHKit.config.use_format :pretty
+```
+
+However, if you prefer non colored text you can use the `:simpletext` formatter. If you want minimal output,
+there is also a `:dot` formatter which will simply output red or green dots based on the success or failure of commands.
+There is also a `:blackhole` formatter which does not output anything.
+
+By default, formatters log to `$stdout`, but they can be constructed with any object which implements `<<`
+for example any `IO` subclass, `String`, `Logger` etc:
+
+```ruby
+# Output to a String:
+output = String.new
+SSHKit.config.output = SSHKit::Formatter::Pretty.new(output)
+# Do something with output
+
+# Or output to a file:
+SSHKit.config.output = SSHKit::Formatter::SimpleText.new(File.open('log/deploy.log', 'wb'))
+```
+
+#### Output Colors
+
+By default, SSHKit will color the output using ANSI color escape sequences
+if the output you are using is associated with a terminal device (tty).
+This means that you should see colors if you are writing output to the terminal (the default),
+but you shouldn't see ANSI color escape sequences if you are writing to a file.
+
+Colors are supported for the `Pretty` and `Dot` formatters, but for historical reasons
+the `SimpleText` formatter never shows colors.
+
+If you want to force SSHKit to show colors, you can set the `SSHKIT_COLOR` environment variable:
+
+```ruby
+ENV['SSHKIT_COLOR'] = 'TRUE'
 ```
 
-However, if you prefer minimal output, `:dot` format will simply output red or green dots based on the success or failure of operations.
+#### Custom formatters
 
-To output directly to $stdout without any formatting, you can use:
+Want custom output formatting? Here's what you have to do:
+
+1. Write a new formatter class in the `SSHKit::Formatter` module. As an example, check out the default [pretty](https://github.com/capistrano/sshkit/blob/master/lib/sshkit/formatters/pretty.rb) formatter.
+1. Set the output format as described above. E.g. if your new formatter is called `FooBar`:
+
+```ruby
+SSHKit.config.use_format :foobar
+```
+
+If your formatter class takes a second `options` argument in its constructor, you can pass options to it like this:
 
 ```ruby
-SSHKit.config.output = $stdout
+SSHKit.config.use_format :foobar, :my_option => "value"
+```
+
+Which will call your constructor:
+
+```ruby
+SSHKit::Formatter::FooBar.new($stdout, :my_option => "value")
 ```
 
 ## Output Verbosity
@@ -226,6 +472,19 @@ and defaults to `Logger::INFO`.
 
 At present the `Logger::WARN`, `ERROR` and `FATAL` are not used.
 
+## Deprecation warnings
+
+Deprecation warnings are logged directly to `stderr` by default. This behaviour
+can be changed by setting the `SSHKit.config.deprecation_output` option:
+
+```ruby
+# Disable deprecation warnings
+SSHKit.config.deprecation_output = nil
+
+# Log deprecation warnings to a file
+SSHKit.config.deprecation_output = File.open('log/deprecation_warnings.log', 'wb')
+```
+
 ## Connection Pooling
 
 SSHKit uses a simple connection pool (enabled by default) to reduce the
@@ -249,6 +508,12 @@ pooling behaviour entirely by setting the idle_timeout to zero:
 SSHKit::Backend::Netssh.pool.idle_timeout = 0 # disabled
 ```
 
+## Tunneling and other related SSH themes
+
+In order to do special gymnasitcs with SSH, tunneling, aliasing, complex options, etc with SSHKit it is possible to use [the underlying Net::SSH API](https://github.com/capistrano/sshkit/blob/master/EXAMPLES.md#setting-global-ssh-options) however in many cases it is preferred to use the system SSH configuration file at [`~/.ssh/config`](http://man.cx/ssh_config). This allows you to have personal configuration tied to your machine that does not have to be committed with the repository. If this is not suitable (everyone on the team needs a proxy command, or some special aliasing) a file in the same format can be placed in the project directory at `~/yourproject/.ssh/config`, this will be merged with the system settings in `~/.ssh/config`, and with any configuration specified in [`SSHKit::Backend::Netssh.config.ssh_options`](https://github.com/capistrano/sshkit/blob/master/lib/sshkit/backends/netssh.rb#L133).
+
+These system level files are the preferred way of setting up tunneling and proxies because the system implementations of these things are faster and better than the Ruby implementations you would get if you were to configure them through Net::SSH. In cases where it's not possible (Windows?), it should be possible to make use of the Net::SSH APIs to setup tunnels and proxy commands before deferring control to Capistrano/SSHKit..
+
 ## SSHKit Related Blog Posts
 
 [SSHKit Gem Basics](http://www.rubyplus.com/articles/591)

data/RELEASING.md

@@ -1,10 +1,18 @@
 # Releasing
 
-* **Ensure the tests are passing.**
-* Determine which would be the correct next version number according to [semver](http://semver.org/).
-* Update the version in `./lib/sshkit/version.rb`.
-* Update the `CHANGELOG`.
-* Commit the changelog and version in a single commit, the message should be "Preparing vX.Y.Z"
-* Tag the commit `git tag vX.Y.Z` (if tagging a historical commit, `git tag` can take a *SHA1* after the tag name)
-* Push new commits, and tags to Github.
-* Push the gem to [rubygems](http://rubygems.org).
+## Prerequisites
+
+* You must have commit rights to the SSHKit repository.
+* You must have push rights for the sshkit gem on rubygems.org.
+* You must be using Ruby >= 2.1.0.
+* Your `~/.netrc` must be configured with your GitHub credentials, [as explained here](https://github.com/mattbrictson/chandler#2-configure-netrc).
+
+## How to release
+
+1. Run `bundle install` to make sure that you have all the gems necessary for testing and releasing.
+2.  **Ensure the tests are passing by running `rake test`.** If functional tests fail, ensure you have [Vagrant](https://www.vagrantup.com) installed and have started it with `vagrant up`.
+3. Determine which would be the correct next version number according to [semver](http://semver.org/).
+4. Update the version in `./lib/sshkit/version.rb`.
+5. Update the `CHANGELOG`.
+6. Commit the changelog and version in a single commit, the message should be "Preparing vX.Y.Z"
+7. Run `rake release`; this will tag, push to GitHub, publish to rubygems.org, and upload the latest changelog entry to the [GitHub releases page](https://github.com/capistrano/sshkit/releases).

data/Rakefile

@@ -25,3 +25,11 @@ end
 Rake::Task["test:functional"].enhance do
   warn "Remember there are still some VMs running, kill them with `vagrant halt` if you are finished using them."
 end
+
+task "release:rubygem_push" do
+  # Delay loading Chandler until this point, since it requires Ruby >= 2.1,
+  # which may not be available in all environments (e.g. Travis).
+  # We assume that the person doing `rake release` has Ruby >= 2.1.
+  require "chandler/tasks"
+  Rake.application.invoke_task("chandler:push")
+end

data/lib/sshkit.rb

@@ -6,15 +6,6 @@ module SSHKit
 
     attr_accessor :config
 
-    def capture_output(io, &block)
-      original_io = config.output
-      config.output = io
-      config.output.extend(SSHKit::Utils::CaptureOutputMethods)
-      yield
-    ensure
-      config.output = original_io
-    end
-
     def configure
       @@config ||= Configuration.new
       yield config

data/lib/sshkit/all.rb

@@ -9,15 +9,19 @@ require_relative 'command_map'
 require_relative 'configuration'
 require_relative 'coordinator'
 
+require_relative 'deprecation_logger'
+
 require_relative 'exception'
 
 require_relative 'logger'
 require_relative 'log_message'
 
+require_relative 'mapping_interaction_handler'
+
 require_relative 'formatters/abstract'
 require_relative 'formatters/black_hole'
-require_relative 'formatters/simple_text'
 require_relative 'formatters/pretty'
+require_relative 'formatters/simple_text'
 require_relative 'formatters/dot'
 
 require_relative 'runners/abstract'
@@ -31,6 +35,4 @@ require_relative 'backends/connection_pool'
 require_relative 'backends/printer'
 require_relative 'backends/netssh'
 require_relative 'backends/local'
-require_relative 'backends/skipper'
-
-require_relative 'utils/capture_output_methods'
+require_relative 'backends/skipper'

data/lib/sshkit/backends/abstract.rb

@@ -6,10 +6,13 @@ module SSHKit
 
     class Abstract
 
+      extend Forwardable
+      def_delegators :output, :log, :fatal, :error, :warn, :info, :debug
+
       attr_reader :host
 
       def run
-        # Nothing to do
+        instance_exec(@host, &@block)
       end
 
       def initialize(host, &block)
@@ -18,55 +21,39 @@ module SSHKit
         @block = block
       end
 
-      def log(messages)
-        info(messages)
-      end
-
-      def fatal(messages)
-        output << LogMessage.new(Logger::FATAL, messages)
-      end
-
-      def error(messages)
-        output << LogMessage.new(Logger::ERROR, messages)
-      end
-
-      def warn(messages)
-        output << LogMessage.new(Logger::WARN, messages)
-      end
-
-      def info(messages)
-        output << LogMessage.new(Logger::INFO, messages)
-      end
-
-      def debug(messages)
-        output << LogMessage.new(Logger::DEBUG, messages)
-      end
-
-      def trace(messages)
-        output << LogMessage.new(Logger::TRACE, messages)
-      end
-
       def make(commands=[])
-        raise MethodUnavailableError
+        execute :make, commands
       end
 
       def rake(commands=[])
-        raise MethodUnavailableError
+        execute :rake, commands
       end
 
-      def test(command, args=[])
-        raise MethodUnavailableError
+      def test(*args)
+        options = args.extract_options!.merge(raise_on_non_zero_exit: false, verbosity: Logger::DEBUG)
+        create_command_and_execute(args, options).success?
       end
 
-      def execute(command, args=[])
-        raise MethodUnavailableError
+      def capture(*args)
+        options = { verbosity: Logger::DEBUG, strip: true }.merge(args.extract_options!)
+        result = create_command_and_execute(args, options).full_stdout
+        options[:strip] ? result.strip : result
       end
 
-      def capture(command, args=[])
-        raise MethodUnavailableError
+      def background(*args)
+        SSHKit.config.deprecation_logger.log(
+          'The background method is deprecated. Blame badly behaved pseudo-daemons!'
+        )
+        options = args.extract_options!.merge(run_in_background: true)
+        create_command_and_execute(args, options).success?
+      end
+
+      def execute(*args)
+        options = args.extract_options!
+        create_command_and_execute(args, options).success?
       end
 
-      def within(directory, &block)
+      def within(directory, &_block)
         (@pwd ||= []).push directory.to_s
         execute <<-EOTEST, verbosity: Logger::DEBUG
           if test ! -d #{File.join(@pwd)}
@@ -79,7 +66,7 @@ module SSHKit
         @pwd.pop
       end
 
-      def with(environment, &block)
+      def with(environment, &_block)
         @_env = (@env ||= {})
         @env = @_env.merge environment
         yield
@@ -88,7 +75,7 @@ module SSHKit
         remove_instance_variable(:@_env)
       end
 
-      def as(who, &block)
+      def as(who, &_block)
         if who.is_a? Hash
           @user  = who[:user]  || who["user"]
           @group = who[:group] || who["group"]
@@ -118,10 +105,23 @@ module SSHKit
         end
       end
 
+      # Backends which extend the Abstract backend should implement the following methods:
+      def upload!(_local, _remote, _options = {}) raise MethodUnavailableError end
+      def download!(_remote, _local=nil, _options = {}) raise MethodUnavailableError end
+      def execute_command(_cmd) raise MethodUnavailableError end
+      private :execute_command # Can inline after Ruby 2.1
+
       private
 
-      def command(*args)
-        options = args.extract_options!
+      def output
+        SSHKit.config.output
+      end
+
+      def create_command_and_execute(args, options)
+        command(args, options).tap { |cmd| execute_command(cmd) }
+      end
+
+      def command(args, options)
         SSHKit::Command.new(*[*args, options.merge({in: @pwd.nil? ? nil : File.join(@pwd), env: @env, host: @host, user: @user, group: @group})])
       end