ridley 0.10.0.rc1 → 0.10.0.rc2

data/bootstrappers/windows_omnibus.erb

@@ -25,10 +25,6 @@ setlocal
 <%= "SETX HTTP_PROXY \"#{bootstrap_proxy}\"" if bootstrap_proxy %>
 mkdir <%= bootstrap_directory %>
 
-> <%= bootstrap_directory %>\wget.vbs (
- <%= windows_wget_vb %>
-)
-
 > <%= bootstrap_directory %>\wget.ps1 (
  <%= windows_wget_powershell %>
 )
@@ -87,24 +83,7 @@ set "REMOTE_SOURCE_MSI_URL=https://www.opscode.com/chef/download?p=windows&pv=%M
 set "LOCAL_DESTINATION_MSI_PATH=<%= local_download_path %>"
 set "FALLBACK_QUERY_STRING=&DownloadContext=PowerShell"
 
-cscript /nologo <%= bootstrap_directory %>\wget.vbs /url:"%REMOTE_SOURCE_MSI_URL%" /path:"%LOCAL_DESTINATION_MSI_PATH%"
-
-rem Work around issues found in Windows Server 2012 around job objects not respecting WSMAN memory quotas
-rem that cause the MSI download process to exceed the quota even when it is increased by administrators.
-rem Retry the download using a more memory-efficient mechanism that only works if PowerShell is available.
-if ERRORLEVEL 1 (      
-    echo Warning: Failed to download "%REMOTE_SOURCE_MSI_URL%" to "%LOCAL_DESTINATION_MSI_PATH%"
-    echo Warning: Retrying download with PowerShell if available
-    if EXIST "%LOCAL_DESTINATION_MSI_PATH%" del /f /q "%LOCAL_DESTINATION_MSI_PATH%"
-    powershell -ExecutionPolicy Unrestricted -NoProfile -NonInteractive "& '<%= bootstrap_directory %>\wget.ps1' '%REMOTE_SOURCE_MSI_URL%%FALLBACK_QUERY_STRING%' '%LOCAL_DESTINATION_MSI_PATH%'"
-
-    if NOT ERRORLEVEL 1 (
-        echo Download succeeded
-    ) else (
-        echo Failed to download "%REMOTE_SOURCE_MSI_URL%"
-        echo Subsequent attempt to install the downloaded MSI is likely to fail
-    )
-)
+powershell -ExecutionPolicy Unrestricted -NoProfile -NonInteractive "& '<%= bootstrap_directory %>\wget.ps1' '%REMOTE_SOURCE_MSI_URL%%FALLBACK_QUERY_STRING%' '%LOCAL_DESTINATION_MSI_PATH%'"
 
 <%= install_chef %>
 

data/lib/ridley/bootstrap_bindings/windows_template_binding.rb

@@ -119,59 +119,6 @@ CONFIG
       raise Errors::EncryptedDataBagSecretNotFound, "Error bootstrapping: Encrypted data bag secret provided but not found at '#{encrypted_data_bag_secret_path}'"      
     end
 
-    # Implements a Visual Basic script that attempts a simple
-    # 'wget' to download the Chef msi
-    # 
-    # @return [String]
-    def windows_wget_vb
-      win_wget = <<-WGET
-url = WScript.Arguments.Named("url")
-path = WScript.Arguments.Named("path")
-proxy = null
-Set objXMLHTTP = CreateObject("MSXML2.ServerXMLHTTP")
-Set wshShell = CreateObject( "WScript.Shell" )
-Set objUserVariables = wshShell.Environment("USER")
-
-'http proxy is optional
-'attempt to read from HTTP_PROXY env var first
-On Error Resume Next
-
-If NOT (objUserVariables("HTTP_PROXY") = "") Then
-proxy = objUserVariables("HTTP_PROXY")
-
-'fall back to named arg
-ElseIf NOT (WScript.Arguments.Named("proxy") = "") Then
-proxy = WScript.Arguments.Named("proxy")
-End If
-
-If NOT isNull(proxy) Then
-'setProxy method is only available on ServerXMLHTTP 6.0+
-Set objXMLHTTP = CreateObject("MSXML2.ServerXMLHTTP.6.0")
-objXMLHTTP.setProxy 2, proxy
-End If
-
-On Error Goto 0
-
-objXMLHTTP.open "GET", url, false
-objXMLHTTP.send()
-If objXMLHTTP.Status = 200 Then
-Set objADOStream = CreateObject("ADODB.Stream")
-objADOStream.Open
-objADOStream.Type = 1
-objADOStream.Write objXMLHTTP.ResponseBody
-objADOStream.Position = 0
-Set objFSO = Createobject("Scripting.FileSystemObject")
-If objFSO.Fileexists(path) Then objFSO.DeleteFile path
-Set objFSO = Nothing
-objADOStream.SaveToFile path
-objADOStream.Close
-Set objADOStream = Nothing
-End if
-Set objXMLHTTP = Nothing
-WGET
-      escape_and_echo(win_wget)
-    end
-
     # Implements a Powershell script that attempts a simple
     # 'wget' to download the Chef msi
     # 

data/lib/ridley/bootstrapper.rb

@@ -4,14 +4,11 @@ module Ridley
     autoload :Context, 'ridley/bootstrapper/context'
 
     include Celluloid
-    include Celluloid::Logger
-    
+    include Ridley::Logging
+
     # @return  [Array<String>]
     attr_reader :hosts
 
-    # @return [Array<Bootstrapper::Context>]
-    attr_reader :contexts
-
     # @return [Hash]
     attr_reader :options
 
@@ -47,7 +44,7 @@ module Ridley
     # @option options [String] :template
     #   bootstrap template to use
     def initialize(hosts, options = {})
-      @hosts         = Array(hosts).collect(&:to_s).uniq
+      @hosts         = Array(hosts).flatten.collect(&:to_s).uniq
       @options       = options.dup
       @options[:ssh] ||= Hash.new
       @options[:ssh] = {
@@ -56,16 +53,22 @@ module Ridley
       }.merge(@options[:ssh])
 
       @options[:sudo] = @options[:ssh][:sudo]
-      @contexts = @hosts.collect do |host|
-        Context.create(host, options)
-      end
     end
 
+    # @raise [Errors::HostConnectionError] if a node is unreachable
+    #
+    # @return [Array<Bootstrapper::Context>]
+    def contexts
+      @contexts ||= @hosts.collect { |host| Context.create(host, options) }
+    end
+
+    # @raise [Errors::HostConnectionError] if a node is unreachable
+    #
     # @return [HostConnector::ResponseSet]
     def run
       workers = Array.new
       futures = contexts.collect do |context|
-        info "Running bootstrap command on #{context.host}"
+        log.info { "Running bootstrap command on #{context.host}" }
 
         workers << worker = context.host_connector::Worker.new(context.host, self.options.freeze)
 

data/lib/ridley/bootstrapper/context.rb

@@ -5,7 +5,6 @@ module Ridley
     # @author Jamie Winsor <reset@riotgames.com>
     class Context
       class << self
-
         # @param [String] host
         # @option options [Hash] :ssh
         #   * :user (String) a shell user that will login to each node and perform the bootstrap command on (required)
@@ -38,6 +37,8 @@ module Ridley
         #   bootstrap with sudo (default: true)
         # @option options [String] :template ('omnibus')
         #   bootstrap template to use
+        #
+        # @raise [Errors::HostConnectionError] if a node is unreachable
         def create(host, options = {})
           host_connector = HostConnector.best_connector_for(host, options)
           template_binding = case host_connector.to_s

data/lib/ridley/errors.rb

@@ -40,6 +40,9 @@ module Ridley
     class EncryptedDataBagSecretNotFound < BootstrapError; end
     class HostConnectionError < BootstrapError; end
 
+    class RemoteCommandError < RidleyError; end
+    class RemoteScriptError < RemoteCommandError; end
+
     # Exception thrown when the maximum amount of requests is exceeded.
     class RedirectLimitReached < RidleyError
       attr_reader :response

data/lib/ridley/host_connector.rb

@@ -14,24 +14,32 @@ module Ridley
 
     class << self
       # Finds and returns the best HostConnector for a given host
-      # 
+      #
       # @param  host [String]
       #   the host to attempt to connect to
       # @option options [Hash] :ssh
       #   * :port (Fixnum) the ssh port to connect on the node the bootstrap will be performed on (22)
       # @option options [Hash] :winrm
       #   * :port (Fixnum) the winrm port to connect on the node the bootstrap will be performed on (5985)
-      # 
+      # @param block [Proc]
+      #   an optional block that is yielded the best HostConnector
+      #
       # @return [Ridley::HostConnector] a class under Ridley::HostConnector
-      def best_connector_for(host, options = {})
+      def best_connector_for(host, options = {}, &block)
         ssh_port, winrm_port = parse_port_options(options)
         if connector_port_open?(host, ssh_port)
-          Ridley::HostConnector::SSH
+          host_connector = Ridley::HostConnector::SSH
         elsif connector_port_open?(host, winrm_port)
-          Ridley::HostConnector::WinRM
+          host_connector = Ridley::HostConnector::WinRM
         else
           raise Ridley::Errors::HostConnectionError, "No available connection method available on #{host}."
         end
+
+        if block_given?
+          yield host_connector
+        else
+          host_connector
+        end
       end
 
       # Checks to see if the given port is open for TCP connections
@@ -41,19 +49,16 @@ module Ridley
       #   the host to attempt to connect to
       # @param  port [Fixnum]
       #   the port to attempt to connect on
-      # 
+      #
       # @return [Boolean]
       def connector_port_open?(host, port)
-        Timeout::timeout(1) do
-          begin
-            socket = TCPSocket.new(host, port)
-            socket.close
-            true
-          rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH
-            false
-          end
+        Timeout::timeout(3) do
+          socket = TCPSocket.new(host, port)
+          socket.close
         end
-      rescue Timeout::Error
+
+        true
+      rescue Timeout::Error, SocketError, Errno::ECONNREFUSED, Errno::EHOSTUNREACH
         false
       end
 
@@ -64,7 +69,7 @@ module Ridley
       #   * :port (Fixnum) the ssh port to connect on the node the bootstrap will be performed on (22)
       # @option options [Hash] :winrm
       #   * :port (Fixnum) the winrm port to connect on the node the bootstrap will be performed on (5985)
-      # 
+      #
       # @return [Array]
       def parse_port_options(options)
         ssh_port = options[:ssh][:port] if options[:ssh]

data/lib/ridley/host_connector/ssh/worker.rb

@@ -12,11 +12,13 @@ module Ridley
         attr_reader :host
         # @return [Hashie::Mash]
         attr_reader :options
+
+        EMBEDDED_RUBY_PATH = '/opt/chef/embedded/bin/ruby'.freeze
         
         # @param [Hash] options
         def initialize(host, options = {})
-          @options = options.deep_symbolize_keys
-          @options = options[:ssh] if options[:ssh]
+          options = options.deep_symbolize_keys
+          @options = options[:ssh] || Hash.new
           @host    = host
           @sudo    = @options[:sudo]
           @user    = @options[:user]
@@ -90,6 +92,42 @@ module Ridley
           [ :error, response ]
         end
 
+        # Executes a chef-client command on the nodes
+        #
+        # @return [#run]
+        def chef_client
+          command = "chef-client"
+          if sudo
+            command = "sudo #{command}"
+          end
+
+          run(command)
+        end
+
+        # Executes a copy of the encrypted_data_bag_secret to the nodes
+        #
+        # @param [String] encrypted_data_bag_secret_path
+        #   the path to the encrypted_data_bag_secret
+        # 
+        # @return [#run]
+        def put_secret(encrypted_data_bag_secret_path)
+          secret  = File.read(encrypted_data_bag_secret_path).chomp
+          command = "echo '#{secret}' > /etc/chef/encrypted_data_bag_secret; chmod 0600 /etc/chef/encrypted_data_bag_secret"
+
+          run(command)
+        end
+
+        # Executes a provided Ruby script in the embedded Ruby installation
+        #
+        # @param [Array<String>] command_lines
+        #   An Array of lines of the command to be executed
+        # 
+        # @return [#run]
+        def ruby_script(command_lines)
+          command = "#{EMBEDDED_RUBY_PATH} -e \"#{command_lines.join(';')}\""
+          run(command)
+        end
+
         private
 
           attr_reader :runner

data/lib/ridley/host_connector/winrm.rb

@@ -2,6 +2,7 @@ module Ridley
   module HostConnector
     # @author Kyle Allan <kallan@riotgames.com>
     class WinRM
+      autoload :CommandUploader, 'ridley/host_connector/winrm/command_uploader'
       autoload :Worker, 'ridley/host_connector/winrm/worker'
 
       class << self

data/lib/ridley/host_connector/winrm/command_uploader.rb

@@ -0,0 +1,90 @@
+module Ridley
+  module HostConnector
+    class WinRM
+      # @author Kyle Allan <kallan@riotgames.com>
+      # @author Justin Campbell <justin.campbell@riotgames.com>
+      #
+      # @example
+      #   command_uploader = CommandUploader.new(long_command, winrm)
+      #   command_uploader.upload
+      #   command_uploader.command
+      #
+      #   This class is used by WinRM Workers when the worker is told to execute a command that
+      #   might be too long for WinRM to handle.
+      #
+      #   After an instance of this class is created, the upload method will upload the long command
+      #   to the node being worked on in chunks. Once on the node, some Powershell code will decode
+      #   the long_command into a batch file. The command method will return a String representing the
+      #   command the worker will use to execute the uploaded batch script.
+      class CommandUploader
+        CHUNK_LIMIT = 1024
+
+        # @return [WinRM::WinRMWebService]
+        attr_reader :winrm
+        # @return [String]
+        attr_reader :base64_file_name
+        # @return [String]
+        attr_reader :command_file_name
+
+        # @param [WinRM::WinRMWebService] winrm
+        def initialize(winrm)
+          @winrm             = winrm
+          @base64_file_name  = get_file_path("winrm-upload-base64-#{unique_string}")
+          @command_file_name = get_file_path("winrm-upload-#{unique_string}.bat")
+        end
+
+        # Uploads the command encoded as base64 to a file on the host
+        # and then uses Powershell to transform the base64 file into the
+        # command that was originally passed through.
+        #
+        # @param [String] command_string
+        def upload(command_string)
+          upload_command(command_string)
+          convert_command
+        end
+
+        # @return [String] the command to execute the uploaded file
+        def command
+          "cmd.exe /C #{command_file_name}"
+        end
+
+        # Runs a delete command on the files generated by #base64_file_name
+        # and #command_file_name
+        def cleanup
+          winrm.run_cmd( "del #{base64_file_name} /F /Q" )
+          winrm.run_cmd( "del #{command_file_name} /F /Q" )
+        end
+
+        private
+
+          def upload_command(command_string)
+            command_string_chars(command_string).each_slice(CHUNK_LIMIT) do |chunk|
+              winrm.run_cmd( "echo #{chunk.join} >> \"#{base64_file_name}\"" )
+            end
+          end
+
+          def command_string_chars(command_string)
+            Base64.encode64(command_string).gsub("\n", '').chars.to_a
+          end
+
+          def convert_command
+            winrm.powershell <<-POWERSHELL
+              $base64_string = Get-Content \"#{base64_file_name}\"
+              $bytes  = [System.Convert]::FromBase64String($base64_string)
+              $new_file = [System.IO.Path]::GetFullPath(\"#{command_file_name}\")
+              [System.IO.File]::WriteAllBytes($new_file,$bytes)
+            POWERSHELL
+          end
+
+          def unique_string
+            @unique_string ||= "#{Process.pid}-#{Time.now.to_i}"
+          end
+
+          # @return [String]
+          def get_file_path(file)
+            (winrm.run_cmd("echo %TEMP%\\#{file}"))[:data][0][:stdout].chomp
+          end
+      end
+    end
+  end
+end

data/lib/ridley/host_connector/winrm/worker.rb

@@ -17,6 +17,14 @@ module Ridley
         attr_reader :options
         # @return [String]
         attr_reader :winrm_endpoint
+        # @return [CommandUploader]
+        attr_reader :command_uploader
+        # @return [Array]
+        attr_reader :command_uploaders
+
+        finalizer :finalizer
+
+        EMBEDDED_RUBY_PATH = 'C:\opscode\chef\embedded\bin\ruby'.freeze
 
         # @param  host [String]
         #   the host the worker is going to work on
@@ -25,24 +33,38 @@ module Ridley
         #   * :password (String) the password for the user that will perform the bootstrap
         #   * :port (Fixnum) the winrm port to connect on the node the bootstrap will be performed on (5985)
         def initialize(host, options = {})
-          @options        = options.deep_symbolize_keys
-          @options        = options[:winrm] if options[:winrm]
-          @host           = host
-          @user           = @options[:user]
-          @password       = @options[:password]
-          @winrm_endpoint = "http://#{host}:#{winrm_port}/wsman"
+          options            = options.deep_symbolize_keys
+          @options           = options[:winrm] || Hash.new
+          @host              = host
+          @user              = @options[:user]
+          @password          = @options[:password]
+          @winrm_endpoint    = "http://#{host}:#{winrm_port}/wsman"
+          @command_uploaders = Array.new
+        end
+
+        def finalizer
+          command_uploaders.map(&:cleanup)
         end
 
         def run(command)
-          command = get_command(command)
+          command_uploaders << command_uploader = CommandUploader.new(winrm)
+          command = get_command(command, command_uploader)
 
           response = Ridley::HostConnector::Response.new(host)
           debug "Running WinRM Command: '#{command}' on: '#{host}' as: '#{user}'"
 
           output = winrm.run_cmd(command) do |stdout, stderr|
-            response.stdout += stdout unless stdout.nil?
-            response.stderr += stderr unless stderr.nil?
+            if stdout
+              response.stdout += stdout
+              info "NODE[#{host}] #{stdout}"
+            end
+
+            if stderr
+              response.stderr += stderr unless stderr.nil?
+              info "NODE[#{host}] #{stdout}"
+            end
           end
+
           response.exit_code = output[:exitcode]
 
           case response.exit_code
@@ -62,7 +84,7 @@ module Ridley
           [ :error, response ]
         end
 
-        # @return [::WinRM::WinRMWebService]
+        # @return [WinRM::WinRMWebService]
         def winrm
           ::WinRM::WinRMWebService.new(winrm_endpoint, :plaintext, user: user, pass: password, disable_sspi: true, basic_auth_only: true)
         end
@@ -73,53 +95,51 @@ module Ridley
         end
 
         # Returns the command if it does not break the WinRM command length
-        # limit. Otherwise, we return an execution of the command as a batch file. 
+        # limit. Otherwise, we return an execution of the command as a batch file.
         #
         # @param  command [String]
-        # 
+        #
         # @return [String]
-        def get_command(command)
-          if command.length < 2047
+        def get_command(command, command_uploader)
+          if command.length < CommandUploader::CHUNK_LIMIT
             command
           else
-            debug "Detected a command that was longer than 2047 characters, uploading command as a file to the host."
-            upload_command_to_host(command)
+            debug "Detected a command that was longer than #{CommandUploader::CHUNK_LIMIT} characters, \
+              uploading command as a file to the host."
+            command_uploader.upload(command)
+            command_uploader.command
           end
         end
 
-        private
-
-          # Uploads the command encoded as base64 to a file on the host
-          # and then uses Powershell to transform the base64 file into the
-          # command that was originally passed through.
-          #
-          # @param  command [String]
-          #
-          # @return [String] the command to execute the uploaded file
-          def upload_command_to_host(command)
-            base64_file = "winrm-upload-base64-#{Process.pid}-#{Time.now.to_i}"
-            base64_file_name = get_file_path(base64_file)
-
-            Base64.encode64(command).gsub("\n", '').chars.to_a.each_slice(8000 - base64_file_name.size) do |chunk|
-              out = winrm.run_cmd( "echo #{chunk.join} >> \"#{base64_file_name}\"" )
-            end
-
-            command_file = "winrm-upload-#{Process.pid}-#{Time.now.to_i}.bat"
-            command_file_name = get_file_path(command_file)
-            winrm.powershell <<-POWERSHELL
-              $base64_string = Get-Content \"#{base64_file_name}\"
-              $bytes  = [System.Convert]::FromBase64String($base64_string) 
-              $new_file = [System.IO.Path]::GetFullPath(\"#{command_file_name}\")
-              [System.IO.File]::WriteAllBytes($new_file,$bytes)
-            POWERSHELL
+        # Executes a chef-client run on the nodes
+        #
+        # @return [#run]
+        def chef_client
+          run("chef-client")
+        end
 
-            "cmd.exe /C #{command_file_name}"
-          end
+        # Executes a copy of the encrypted_data_bag_secret to the nodes
+        #
+        # @param [String] encrypted_data_bag_secret_path
+        #   the path to the encrypted_data_bag_secret
+        #
+        # @return [#run]
+        def put_secret(encrypted_data_bag_secret_path)
+          secret  = File.read(encrypted_data_bag_secret_path).chomp
+          command = "echo #{secret} > C:\\chef\\encrypted_data_bag_secret"
+          run(command)
+        end
 
-          # @return [String]
-          def get_file_path(file)
-            (winrm.run_cmd("echo %TEMP%\\#{file}"))[:data][0][:stdout].chomp
-          end
+        # Executes a provided Ruby script in the embedded Ruby installation
+        #
+        # @param [Array<String>] command_lines
+        #   An Array of lines of the command to be executed
+        #
+        # @return [#run]
+        def ruby_script(command_lines)
+          command = "#{EMBEDDED_RUBY_PATH} -e \"#{command_lines.join(';')}\""
+          run(command)
+        end
       end
     end
   end