ridley 0.5.2 → 0.6.0

data/.travis.yml

@@ -4,3 +4,7 @@ rvm:
   - 1.9.2
   - 1.9.3
   - jruby-19mode
+matrix:
+  allow_failures:
+    - rvm: jruby-19mode
+

data/Guardfile

@@ -12,7 +12,7 @@ guard 'yard', stdout: '/dev/null', stderr: '/dev/null' do
   watch(%r{ext/.+\.c})
 end
 
-guard 'rspec', version: 2, cli: "--color --drb --format Fuubar", all_on_start: false, all_after_pass: false do
+guard 'rspec', cli: "--color --drb --format Fuubar", all_on_start: false, all_after_pass: false do
   watch(%r{^spec/unit/.+_spec\.rb$})
   watch(%r{^spec/acceptance/.+_spec\.rb$})
   

data/README.md

@@ -291,22 +291,12 @@ Given the previous example you could set the default node attribute with the `se
 
 ### Node Attributes
 
-Setting the `default[:my_app][:billing][:enabled]` node level default attribute on the node "jwinsor-1"
+Setting the `node[:my_app][:billing][:enabled]` node level attribute on the node "jwinsor-1"
 
     conn = Ridley.connection
     conn.sync do
       obj = node.find("jwinsor-1")
-      obj.set_default_attribute("my_app.billing.enabled", false)
-      obj.save
-    end
-
-Other attribute precedence levels can be set with their own respective set attribute functions
-
-    conn = Ridley.connection
-    conn.sync do
-      obj = node.find("jwinsor-1")
-      obj.set_override_attribute("my_app.proxy.enabled", false)
-      obj.set_normal_attribute("my_app.webapp.enabled", false)
+      obj.set_attribute("my_app.billing.enabled", false)
       obj.save
     end
 

data/bootstrappers/omnibus.erb

@@ -64,6 +64,9 @@ cat <<'EOP'
 EOP
 ) > /etc/chef/client.rb
 
+<%# Remove client pem if it already exists -%>
+rm /etc/chef/client.pem
+
 (
 cat <<'EOP'
 <%= first_boot %>

data/lib/ridley/bootstrapper/context.rb

@@ -146,7 +146,7 @@ CONFIG
 
       # @return [String]
       def first_boot
-        attributes.merge(run_list: run_list).to_json
+        MultiJson.encode attributes.merge(run_list: run_list)
       end
 
       # The validation key to create a new client for the node
@@ -155,7 +155,7 @@ CONFIG
       #
       # @return [String]
       def validation_key
-        IO.read(validator_path).chomp
+        IO.read(File.expand_path(validator_path)).chomp
       rescue Errno::ENOENT
         raise Errors::ValidatorNotFound, "Error bootstrapping: Validator not found at '#{validator_path}'"
       end

data/lib/ridley/bootstrapper.rb

@@ -25,43 +25,45 @@ module Ridley
     attr_reader :contexts
 
     # @return [Hash]
-    attr_reader :ssh_config
+    attr_reader :options
 
     # @param [Array<#to_s>] hosts
-    # @option options [String] :ssh_user
-    # @option options [String] :ssh_password
-    # @option options [Array<String>, String] :ssh_keys
-    # @option options [Float] :ssh_timeout
-    #   timeout value for SSH bootstrap (default: 1.5)
+    # @option options [Hash] :ssh
+    #   * :user (String) a shell user that will login to each node and perform the bootstrap command on (required)
+    #   * :password (String) the password for the shell user that will perform the bootstrap
+    #   * :keys (Array, String) an array of keys (or a single key) to authenticate the ssh user with instead of a password
+    #   * :timeout (Float) [5.0] timeout value for SSH bootstrap
     # @option options [String] :validator_client
     # @option options [String] :validator_path
     #   filepath to the validator used to bootstrap the node (required)
-    # @option options [String] :bootstrap_proxy
-    #   URL to a proxy server to bootstrap through (default: nil)
-    # @option options [String] :encrypted_data_bag_secret_path
-    #   filepath on your host machine to your organizations encrypted data bag secret (default: nil)
-    # @option options [Hash] :hints
-    #   a hash of Ohai hints to place on the bootstrapped node (default: Hash.new)
-    # @option options [Hash] :attributes
-    #   a hash of attributes to use in the first Chef run (default: Hash.new)
-    # @option options [Array] :run_list
-    #   an initial run list to bootstrap with (default: Array.new)
-    # @option options [String] :chef_version
-    #   version of Chef to install on the node (default: {Ridley::CHEF_VERSION})
-    # @option options [String] :environment
-    #   environment to join the node to (default: '_default')
-    # @option options [Boolean] :sudo
+    # @option options [String] :bootstrap_proxy (nil)
+    #   URL to a proxy server to bootstrap through
+    # @option options [String] :encrypted_data_bag_secret_path (nil)
+    #   filepath on your host machine to your organizations encrypted data bag secret
+    # @option options [Hash] :hints (Hash.new)
+    #   a hash of Ohai hints to place on the bootstrapped node
+    # @option options [Hash] :attributes (Hash.new)
+    #   a hash of attributes to use in the first Chef run
+    # @option options [Array] :run_list (Array.new)
+    #   an initial run list to bootstrap with
+    # @option options [String] :chef_version (Ridley::CHEF_VERSION)
+    #   version of Chef to install on the node
+    # @option options [String] :environment ('_default')
+    #   environment to join the node to
+    # @option options [Boolean] :sudo (true)
     #   bootstrap with sudo (default: true)
-    # @option options [String] :template
-    #   bootstrap template to use (default: omnibus)
+    # @option options [String] :template ('omnibus')
+    #   bootstrap template to use
     def initialize(hosts, options = {})
-      @hosts      = Array(hosts).collect(&:to_s).uniq
-      @ssh_config = {
-        user: options.fetch(:ssh_user),
-        password: options[:ssh_password],
-        keys: options[:ssh_keys],
-        timeout: (options[:ssh_timeout] || 1.5)
-      }
+      @hosts         = Array(hosts).collect(&:to_s).uniq
+      @options       = options.dup
+      @options[:ssh] ||= Hash.new
+      @options[:ssh] = {
+        timeout: 5.0,
+        sudo: true
+      }.merge(@options[:ssh])
+
+      @options[:sudo] = @options[:ssh][:sudo]
 
       @contexts = @hosts.collect do |host|
         Context.new(host, options)
@@ -70,30 +72,22 @@ module Ridley
 
     # @return [SSH::ResponseSet]
     def run
-      if contexts.length >= 2
-        pool = SSH::Worker.pool(size: contexts.length, args: [self.ssh_config])
-      else
-        pool = SSH::Worker.new(self.ssh_config)
-      end
+      workers = Array.new
+      futures = contexts.collect do |context|
+        info "Running bootstrap command on #{context.host}"
 
-      responses = contexts.collect do |context|
-        pool.future.run(context.host, context.boot_command)
-      end.collect(&:value)
+        workers << worker = SSH::Worker.new_link(self.options[:ssh].freeze)
+        worker.future.run(context.host, context.boot_command)
+      end
 
       SSH::ResponseSet.new.tap do |response_set|
-        responses.each do |message|
-          status, response = message
-
-          case status
-          when :ok
-            response_set.add_ok(response)
-          when :error
-            response_set.add_error(response)
-          end
+        futures.each do |future|
+          status, response = future.value
+          response_set.add_response(response)
         end
       end
     ensure
-      pool.terminate if pool
+      workers.map(&:terminate)
     end
   end
 end

data/lib/ridley/connection.rb

@@ -3,7 +3,10 @@ module Ridley
   class Connection
     class << self
       def sync(options, &block)
-        new(options).sync(&block)
+        conn = new(options)
+        conn.sync(&block)
+      ensure
+        conn.terminate if conn && conn.alive?
       end
       
       # @raise [ArgumentError]
@@ -16,6 +19,12 @@ module Ridley
           missing.collect! { |opt| "'#{opt}'" }
           raise ArgumentError, "Missing required option(s): #{missing.join(', ')}"
         end
+
+        missing_values = options.slice(*REQUIRED_OPTIONS).select { |key, value| !value.present? }
+        unless missing_values.empty?
+          values = missing_values.keys.collect { |opt| "'#{opt}'" }
+          raise ArgumentError, "Missing value for required option(s): '#{values.join(', ')}'"
+        end
       end
 
       # A hash of default options to be used in the Connection initializer
@@ -30,17 +39,20 @@ module Ridley
     end
 
     extend Forwardable
+
+    include Celluloid
     include Ridley::DSL
+    include Ridley::Logging
 
-    attr_reader :client_name
-    attr_reader :client_key
     attr_reader :organization
-    attr_reader :ssh
 
-    attr_reader :validator_client
-    attr_reader :validator_path
-    attr_reader :encrypted_data_bag_secret_path
+    attr_accessor :client_name
+    attr_accessor :client_key
+    attr_accessor :validator_client
+    attr_accessor :validator_path
+    attr_accessor :encrypted_data_bag_secret_path
 
+    attr_accessor :ssh
     attr_accessor :thread_count
 
     def_delegator :conn, :build_url
@@ -60,6 +72,18 @@ module Ridley
 
     def_delegator :conn, :in_parallel
 
+    OPTIONS = [
+      :server_url,
+      :client_name,
+      :client_key,
+      :organization,
+      :validator_client,
+      :validator_path,
+      :encrypted_data_bag_secret_path,
+      :thread_count,
+      :ssl
+    ].freeze
+
     REQUIRED_OPTIONS = [
       :server_url,
       :client_name,
@@ -73,20 +97,20 @@ module Ridley
     # @option options [String] :client_name
     #   name of the client used to authenticate with the Chef API
     # @option options [String] :client_key
-    #   filepath to the client's private key used to authenticate with
-    #   the Chef API
+    #   filepath to the client's private key used to authenticate with the Chef API
     # @option options [String] :organization
     #   the Organization to connect to. This is only used if you are connecting to
     #   private Chef or hosted Chef
-    # @option options [String] :validator_client
-    #   (default: nil)
-    # @option options [String] :validator_path
-    #   (default: nil)
-    # @option options [String] :encrypted_data_bag_secret_path
-    #   (default: nil)
-    # @option options [Integer] :thread_count
-    # @option options [Hash] :ssh
-    #   authentication credentials for bootstrapping or connecting to nodes (default: Hash.new)
+    # @option options [String] :validator_client (nil)
+    # @option options [String] :validator_path (nil)
+    # @option options [String] :encrypted_data_bag_secret_path (nil)
+    # @option options [Integer] :thread_count (DEFAULT_THREAD_COUNT)
+    # @option options [Hash] :ssh (Hash.new)
+    #   * :user (String) a shell user that will login to each node and perform the bootstrap command on (required)
+    #   * :password (String) the password for the shell user that will perform the bootstrap
+    #   * :keys (Array, String) an array of keys (or a single key) to authenticate the ssh user with instead of a password
+    #   * :timeout (Float) [5.0] timeout value for SSH bootstrap
+    #   * :sudo (Boolean) [true] bootstrap with sudo
     # @option options [Hash] :params
     #   URI query unencoded key/value pairs
     # @option options [Hash] :headers
@@ -94,10 +118,18 @@ module Ridley
     # @option options [Hash] :request
     #   request options
     # @option options [Hash] :ssl
-    #   SSL options
+    #   * :verify (Boolean) [true] set to false to disable SSL verification
     # @option options [URI, String, Hash] :proxy
     #   URI, String, or Hash of HTTP proxy options
     def initialize(options = {})
+      log.info { "Ridley starting..." }
+      configure(options)
+    end
+
+    # Configure this instance of Ridley::Connection
+    #
+    # @param [Hash] options
+    def configure(options)
       options = self.class.default_options.merge(options)
       self.class.validate_options(options)
 
@@ -180,6 +212,10 @@ module Ridley
       raise Errors::EncryptedDataBagSecretNotFound, "Encrypted data bag secret provided but not found at '#{encrypted_data_bag_secret_path}'"
     end
 
+    def finalize
+      log.info { "Ridley stopping..." }
+    end
+
     private
 
       attr_reader :conn

data/lib/ridley/logging.rb

@@ -19,6 +19,7 @@ module Ridley
       def set_logger(obj)
         @logger = (obj.nil? ? Logger.new('/dev/null') : obj)
       end
+      alias_method :logger=, :set_logger
     end
 
     # @return [Logger]

data/lib/ridley/resource.rb

@@ -234,13 +234,26 @@ module Ridley
     #   if the resource does not pass validations
     #
     # @return [Boolean]
-    #   true if successful and false for failure
     def save
       raise Errors::InvalidResource.new(self.errors) unless valid?
 
       self.attributes = self.class.create(connection, self).attributes
       true
     rescue Errors::HTTPConflict
+      self.update
+      true
+    end
+
+    # Updates the instantiated resource on the target remote with any changes made
+    # to self
+    #
+    # @raise [Errors::InvalidResource]
+    #   if the resource does not pass validations
+    #
+    # @return [Boolean]
+    def update
+      raise Errors::InvalidResource.new(self.errors) unless valid?
+
       self.attributes = self.class.update(connection, self).attributes
       true
     end

data/lib/ridley/resources/node.rb

@@ -5,11 +5,11 @@ module Ridley
       # @overload bootstrap(connection, nodes, options = {})
       #   @param [Ridley::Connection] connection
       #   @param [Array<String>, String] nodes
-      #   @option options [String] :ssh_user
-      #   @option options [String] :ssh_password
-      #   @option options [Array<String>, String] :ssh_keys
-      #   @option options [Float] :ssh_timeout
-      #     timeout value for SSH bootstrap (default: 1.5)
+      #   @param [Hash] ssh
+      #     * :user (String) a shell user that will login to each node and perform the bootstrap command on (required)
+      #     * :password (String) the password for the shell user that will perform the bootstrap
+      #     * :keys (Array, String) an array of keys (or a single key) to authenticate the ssh user with instead of a password
+      #     * :timeout (Float) [5.0] timeout value for SSH bootstrap
       #   @option options [String] :validator_client
       #   @option options [String] :validator_path
       #     filepath to the validator used to bootstrap the node (required)
@@ -31,22 +31,40 @@ module Ridley
       #     bootstrap with sudo (default: true)
       #   @option options [String] :template
       #     bootstrap template to use (default: omnibus)
+      #
+      # @return [SSH::ResponseSet]
       def bootstrap(connection, *args)
-        options = args.last.is_a?(Hash) ? args.pop : Hash.new
+        options = args.extract_options!
 
         default_options = {
           server_url: connection.server_url,
-          ssh_user: connection.ssh[:user],
-          ssh_password: connection.ssh[:password],
-          ssh_timeout: connection.ssh[:timeout],
           validator_path: connection.validator_path,
           validator_client: connection.validator_client,
-          encrypted_data_bag_secret_path: connection.encrypted_data_bag_secret_path
+          encrypted_data_bag_secret_path: connection.encrypted_data_bag_secret_path,
+          ssh: connection.ssh
         }
 
         options = default_options.merge(options)
 
-        Bootstrapper.new(args, options).run
+        Bootstrapper.new(*args, options).run
+      end
+
+      # Merges the given data with the the data of the target node on the remote
+      #
+      # @param [Ridley::Cnonection] connection
+      # @param [Ridley::Node, String] target
+      #   node or identifier of the node to merge
+      # @option options [Array] :run_list
+      #   run list items to merge
+      # @option options [Hash] :attributes
+      #   attributes of normal precedence to merge
+      #
+      # @raise [Errors::HTTPNotFound]
+      #   if the target node is not found
+      #
+      # @return [Ridley::Node]
+      def merge_data(connection, target, options = {})
+        find!(connection, target).merge_data(options)
       end
     end
 
@@ -184,12 +202,65 @@ module Ridley
 
     # Run Chef-Client on the instantiated node
     #
+    # @param [Hash] options
+    #   a hash of options to pass to {Ridley::SSH.start}
+    #
     # @return [SSH::Response]
-    def chef_client
-      Ridley::SSH.start(self, connection.ssh) do |ssh|
+    def chef_client(options = {})
+      options = connection.ssh.merge(options)
+
+      Ridley.log.debug "Running Chef Client on: #{self.public_hostname}"
+      Ridley::SSH.start(self, options) do |ssh|
         ssh.run("sudo chef-client").first
       end
     end
+
+    # Put the connection's encrypted data bag secret onto the instantiated node. If no
+    # encrypted data bag key path is set on the resource's connection then nil will be
+    # returned
+    #
+    # @param [Hash] options
+    #   a hash of options to pass to {Ridley::SSH.start}
+    #
+    # @return [SSH::Response, nil]
+    def put_secret(options = {})
+      if connection.encrypted_data_bag_secret_path.nil? ||
+        !File.exists?(connection.encrypted_data_bag_secret_path)
+
+        return nil
+      end
+
+      options = connection.ssh.merge(options)
+      secret  = File.read(connection.encrypted_data_bag_secret_path).chomp
+      command = "echo '#{secret}' > /etc/chef/encrypted_data_bag_secret; chmod 0600 /etc/chef/encrypted_data_bag_secret"
+
+      Ridley.log.debug "Writing Encrypted Data Bag Secret to: #{self.public_hostname}"
+      Ridley::SSH.start(self, options) do |ssh|
+        ssh.run(command).first
+      end
+    end
+
+    # Merges the instaniated nodes data with the given data and updates
+    # the remote with the merged results
+    #
+    # @option options [Array] :run_list
+    #   run list items to merge
+    # @option options [Hash] :attributes
+    #   attributes of normal precedence to merge
+    #
+    # @return [Ridley::Node]
+    def merge_data(options = {})
+      unless options[:run_list].nil?
+        self.run_list = (self.run_list + Array(options[:run_list])).uniq
+      end
+
+      unless options[:attributes].nil?
+        self.normal = self.normal.deep_merge(options[:attributes])
+      end
+
+      self.update
+      self
+    end
   end
   
   module DSL

data/lib/ridley/resources/sandbox.rb

@@ -3,15 +3,19 @@ module Ridley
     class << self
       # @param [Ridley::Connection] connection
       # @param [Array] checksums
+      # @option options [Integer] :size (12)
+      #   size of the upload pool
       #
       # @return [Ridley::Sandbox]
-      def create(connection, checksums = [])
+      def create(connection, checksums = [], options = {})
+        options.reverse_merge!(size: 12)
+
         sumhash = { checksums: Hash.new }.tap do |chks|
           Array(checksums).each { |chk| chks[:checksums][chk] = nil }
         end
 
         attrs = connection.post("sandboxes", sumhash.to_json).body
-        new(connection, attrs[:sandbox_id], attrs[:checksums])
+        pool(size: options[:size], args: [connection, attrs[:sandbox_id], attrs[:checksums]])
       end
 
       # Checksum the file at the given filepath for a Chef API.
@@ -43,23 +47,51 @@ module Ridley
         end
         digest.hexdigest
       end
+
+      def future(connection, *args)
+        puts connection
+        connection.future(*args)
+      end
     end
 
+    include Celluloid
+
     attr_reader :sandbox_id
     attr_reader :checksums
 
     def initialize(connection, id, checksums)
       @connection = connection
       @sandbox_id = id
-      @checksums = checksums
+      @checksums  = checksums
     end
 
     def checksum(chk_id)
       checksums.fetch(chk_id.to_sym)
     end
 
+    # Concurrently upload multiple files into a sandbox
+    #
+    # @param [Hash] checksums
+    #   a hash of file checksums and file paths
+    #
+    # @example uploading multiple checksums
+    #
+    #   sandbox.multi_upload(
+    #     "e5a0f6b48d0712382295ff30bec1f9cc" => "/Users/reset/code/rbenv-cookbook/recipes/default.rb",
+    #     "de6532a7fbe717d52020dc9f3ae47dbe" => "/Users/reset/code/rbenv-cookbook/recipes/ohai_plugin.rb"
+    #   )
+    def multi_upload(checksums)
+      checksums.collect do |chk_id, path|
+        future.upload(chk_id, path)
+      end.map(&:value)
+    end
+
+    # Upload one file into the sandbox for the given checksum id
+    #
     # @param [String] chk_id
+    #   checksum of the file being uploaded
     # @param [String] path
+    #   path to the file to upload
     #
     # @return [Hash, nil]
     def upload(chk_id, path)
@@ -85,9 +117,9 @@ module Ridley
       # value of the given checksum.
       conn = connection.send(:conn).dup
 
-      url = URI(checksum[:url])
+      url         = URI(checksum[:url])
       upload_path = url.path
-      url.path = ""
+      url.path    = ""
 
       conn.url_prefix = url.to_s
 
@@ -95,7 +127,7 @@ module Ridley
     end
 
     def commit
-      connection.put("sandboxes/#{sandbox_id}", { is_completed: true }.to_json).body
+      connection.put("sandboxes/#{sandbox_id}", MultiJson.encode(is_completed: true)).body
     end
 
     def to_s

data/lib/ridley/ssh/response.rb

@@ -1,7 +1,22 @@
 module Ridley
   class SSH
     # @author Jamie Winsor <jamie@vialstudios.com>
-    class Response < Struct.new(:stdout, :stderr, :exit_code, :exit_signal)
+    class Response
+      attr_reader :host
+
+      attr_accessor :stdout
+      attr_accessor :stderr
+      attr_accessor :exit_code
+      attr_accessor :exit_signal
+
+      def initialize(host, options = {})
+        @host        = host
+        @stdout      = options[:stdout] || String.new
+        @stderr      = options[:stderr] || String.new
+        @exit_code   = options[:exit_code] || -1
+        @exit_signal = options[:exit_signal] || nil
+      end
+
       # Return true if the response was not successful
       #
       # @return [Boolean]