better-riak-client 1.0.5

data/lib/riak/map_reduce_error.rb

@@ -0,0 +1,7 @@
+require 'riak/util/translation'
+
+module Riak
+  # Raised when an error occurred in the Javascript map-reduce chain.
+  # The message will be the body of the JSON error response.
+  class MapReduceError < StandardError; end
+end

data/lib/riak/node/configuration.rb

@@ -0,0 +1,293 @@
+require 'pathname'
+require 'yaml'
+
+module Riak
+  class Node
+    # The directories (and accessor methods) that will be created
+    # under the generated node.
+    NODE_DIRECTORIES = [:bin, :etc, :log, :data, :ring, :pipe]
+
+    NODE_DIRECTORIES.each do |dir|
+      # Makes accessor methods for all the node directories that
+      # return Pathname objects.
+      class_eval %Q{
+        def #{dir}
+          root + '#{dir}'
+        end
+      }
+    end
+
+    # @return [Hash] the contents of the Erlang environment, which will
+    #   be created into the app.config file.
+    attr_reader :env
+
+    # @return [Hash] the command-line switches for the Erlang virtual
+    #   machine, which will be created into the vm.args file
+    attr_reader :vm
+
+    # @return [Hash] the configuration that was passed to the Node
+    #   when initialized
+    attr_reader :configuration
+
+    # @return [Array<Pathname>] where user Erlang code will be loaded from
+    def erlang_sources
+      env[:riak_kv][:add_paths].map {|p| Pathname.new(p) }
+    end
+
+    # @return [Pathname] where user Javascript code will be loaded from
+    def javascript_source
+      Pathname.new(env[:riak_kv][:js_source_dir])
+    end
+
+    # @return [Fixnum] the size of the ring, i.e. number of partitions
+    def ring_size
+      env[:riak_core][:ring_creation_size]
+    end
+
+    # @return [Fixnum] The port used for handing off data to other nodes.
+    def handoff_port
+      env[:riak_core][:handoff_port]
+    end
+
+    # @return [Fixnum] The port to which the HTTP API is connected.
+    def http_port
+      # We'll only support 0.14 and later, which uses http rather than web_ip/web_port
+      env[:riak_core][:http][0][1]
+    end
+
+    # @return [Fixnum] the port to which the Protocol Buffers API is connected.
+    def pb_port
+      env[:riak_kv][:pb_port]
+    end
+
+    # @return [String] the interface to which the HTTP API is connected
+    def http_ip
+      env[:riak_core][:http][0][0]
+    end
+
+    # @return [String] the interface to which the Protocol Buffers API is connected
+    def pb_ip
+      env[:riak_kv][:pb_ip]
+    end
+
+    # @return [Symbol] the storage backend for Riak Search.
+    def search_backend
+      env[:riak_search][:search_backend]
+    end
+
+    # @return [Symbol] the storage backend for Riak KV.
+    def kv_backend
+      env[:riak_kv][:storage_backend]
+    end
+
+    # @return [String] the name of the Riak node as seen by distributed Erlang
+    #   communication. AKA "-name" in vm.args.
+    def name
+      vm['-name']
+    end
+
+    # @return [String] the cookie/shared secret used for connecting
+    #    a cluster
+    def cookie
+      vm['-setcookie']
+    end
+
+    # The source of the Riak installation from where the {Node} will
+    # be generated. This should point to the directory that contains
+    # the 'riak[search]' and 'riak[search]-admin' scripts.
+    # @return [Pathname] the source Riak installation
+    attr_reader :source
+
+    # The root directory of the {Node}, where all files are placed
+    # after generation.
+    # @return [Pathname] the root directory of the node
+    attr_reader :root
+
+    # The script for starting, stopping and pinging the Node.
+    # @return [Pathname] the path to the control script
+    def control_script
+      @control_script ||= root + 'bin' + control_script_name
+    end
+
+    # The name of the 'riak' or 'riaksearch' control script.
+    # @return [String] 'riak' or 'riaksearch'
+    def control_script_name
+      @control_script_name ||= (source + 'riaksearch').exist? ? 'riaksearch' : 'riak'
+    end
+
+    # The script for controlling non-lifecycle features of Riak like
+    # joining, leaving, status, ringready, etc.
+    # @return [Pathname] the path to the administrative script
+    def admin_script
+      @admin_script ||= root + 'bin' + "#{control_script_name}-admin"
+    end
+
+    # The "manifest" file where the node configuration will be
+    # written.
+    # @return [Pathname] the path to the manifest
+    def manifest
+      root + '.node.yml'
+    end
+
+    protected
+    # Populates the proper node configuration from the input config.
+    def configure(hash)
+      raise ArgumentError, t('source_and_root_required') unless hash[:source] && hash[:root]
+      @configuration = hash
+      configure_paths
+      configure_manifest
+      configure_settings
+      configure_logging
+      configure_data
+      configure_ports(hash[:interface], hash[:min_port])
+      configure_name(hash[:interface])
+    end
+
+    # Reads the manifest if it exists, overrides the passed configuration.
+    def configure_manifest
+      @configuration = YAML.load_file(manifest.to_s) if exist?
+    end
+
+    # Sets the data directories for the various on-disk backends and
+    # the ring state.
+    def configure_data
+      [:bitcask, :eleveldb, :merge_index].each {|k| env[k] ||= {} }
+      env[:bitcask][:data_root] ||= (data + 'bitcask').expand_path.to_s
+      env[:eleveldb][:data_root] ||= (data + 'leveldb').expand_path.to_s
+      env[:merge_index][:data_root] ||= (data + 'merge_index').expand_path.to_s
+      env[:riak_core][:ring_state_dir] ||= ring.expand_path.to_s
+      env[:riak_core][:slide_private_dir] ||= (data + 'slide-data').expand_path.to_s
+      NODE_DIRECTORIES.each do |dir|
+        next if [:ring, :pipe].include?(dir)
+        env[:riak_core][:"platform_#{dir}_dir"] ||= send(dir).to_s
+      end
+    end
+
+    # Sets directories and handlers for logging.
+    def configure_logging
+      if env[:lager]
+        env[:lager][:handlers] = {
+          :lager_console_backend => :info,
+          :lager_file_backend => [
+                                  Tuple[(log+"error.log").expand_path.to_s, :error],
+                                  Tuple[(log+"console.log").expand_path.to_s, :info]
+                                 ]
+        }
+        env[:lager][:crash_log] = (log+"crash.log").to_s
+      else
+        # TODO: Need a better way to detect this, the defaults point
+        # to 1.0-style configs. Maybe there should be some kind of
+        # detection routine.
+        # Use sasl error logger for 0.14.
+        env[:riak_err] ||= {
+          :term_max_size => 65536,
+          :fmt_max_bytes => 65536
+        }
+        env[:sasl] = {
+          :sasl_error_logger => Tuple[:file, (log+"sasl-error.log").expand_path.to_s],
+          :errlog_type => :error,
+          :error_logger_mf_dir => (log+"sasl").expand_path.to_s,
+          :error_logger_mf_maxbytes => 10485760,
+          :error_logger_mf_maxfiles => 5
+        }
+      end
+      vm['-env ERL_CRASH_DUMP'] =  (log + 'erl_crash.dump').to_s
+    end
+
+    # Sets the node name and cookie for distributed Erlang.
+    def configure_name(interface)
+      interface ||= "127.0.0.1"
+      vm["-name"] ||= configuration[:name] || "riak#{rand(1000000).to_s}@#{interface}"
+      vm["-setcookie"] ||= configuration[:cookie] || "#{rand(100000).to_s}_#{rand(1000000).to_s}"
+    end
+
+    # Merges input configuration with the defaults.
+    def configure_settings
+      @env = deep_merge(env.dup, configuration[:env]) if configuration[:env]
+      @vm = vm.merge(configuration[:vm]) if configuration[:vm]
+    end
+
+    # Sets the source directory and root directory of the generated node.
+    def configure_paths
+      @source = Pathname.new(configuration[:source]).expand_path
+      @root = Pathname.new(configuration[:root]).expand_path
+      # Systems like Homebrew and Stow symlink the scripts into $PATH,
+      # but RUNNER_BASE_DIR is not relative to the symlink.
+      if (@source + control_script_name).symlink?
+        @source = (@source + control_script_name).realpath.parent
+      end
+    end
+
+    # Sets ports and interfaces for http, protocol buffers, and handoff.
+    def configure_ports(interface, min_port)
+      interface ||= "127.0.0.1"
+      min_port ||= 8080
+      unless env[:riak_core][:http]
+        env[:riak_core][:http] = [Tuple[interface, min_port]]
+        min_port += 1
+      end
+      env[:riak_core][:http] = env[:riak_core][:http].map {|pair| Tuple[*pair] }
+      env[:riak_kv][:pb_ip] = interface unless env[:riak_kv][:pb_ip]
+      unless env[:riak_kv][:pb_port]
+        env[:riak_kv][:pb_port] = min_port
+        min_port += 1
+      end
+      unless env[:riak_core][:handoff_port]
+        env[:riak_core][:handoff_port] = min_port
+        min_port += 1
+      end
+    end
+
+    # Implements a deep-merge of two {Hash} instances.
+    # @param [Hash] source the original hash
+    # @param [Hash] target the new hash
+    # @return [Hash] a {Hash} whose {Hash} values have also been merged
+    def deep_merge(source, target)
+      source.merge(target) do |key, old_val, new_val|
+        if Hash === old_val && Hash === new_val
+          deep_merge(old_val, new_val)
+        else
+          new_val
+        end
+      end
+    end
+
+    # This class lets us specify that some settings should be emitted
+    # as Erlang tuples, even though the first element is not
+    # necessarily a Symbol.
+    class Tuple < Array; end
+
+    # Recursively converts a {Hash} into an Erlang configuration
+    # string that is appropriate for the app.config file.
+    # @param [Hash] hash a collection of configuration values
+    # @param [Fixnum] depth the current nesting level of
+    #    generation/indentation
+    # @return [String] Erlang proplists in a String for use in
+    #   app.config
+    def to_erlang_config(hash, depth = 1)
+      padding = '    ' * depth
+      parent_padding = '    ' * (depth-1)
+      values = hash.map do |k,v|
+        "{#{k}, #{value_to_erlang(v, depth)}}"
+      end.join(",\n#{padding}")
+      "[\n#{padding}#{values}\n#{parent_padding}]"
+    end
+
+    # Converts a value to an Erlang term. Mutually recursive with
+    # {#to_erlang_config}.
+    def value_to_erlang(v, depth=1)
+      case v
+      when Hash
+        to_erlang_config(v, depth+1)
+      when String
+        "\"#{v}\""
+      when Tuple
+        "{" << v.map {|i| value_to_erlang(i, depth+1) }.join(", ") << "}"
+      when Array
+        "[" << v.map {|i| value_to_erlang(i, depth+1) }.join(", ") << "]"
+      else
+        v.to_s
+      end
+    end
+  end
+end

data/lib/riak/node/console.rb

@@ -0,0 +1,133 @@
+require 'expect'
+require 'pathname'
+require 'riak/util/translation'
+
+if ENV['DEBUG_RIAK_CONSOLE']
+  $expect_verbose = true
+end
+
+module Riak
+  class Node
+    # Eases working with the Erlang console when attached to the Riak
+    # node.
+    class Console
+      include Util::Translation
+
+      # @return [String] the name of the connected node
+      attr_accessor :nodename
+
+      # Opens a {Console} by connecting to the node.
+      # @return [Console] the opened console
+      def self.open(node)
+        new node.pipe, node.name
+      end
+
+      # Creates a {Console} from the IO pipes connected to the node.
+      # @param [String,Pathname] pipedir path to the pipes opened by
+      #   run_erl
+      # @param [String] nodename the name of the node the Console will
+      #   be attached to
+      def initialize(pipedir, nodename)
+        @nodename = nodename
+        @mutex = Mutex.new
+        @prompt = /\(#{Regexp.escape(nodename)}\)\d+>\s*/
+        pipedir = Pathname(pipedir)
+        pipedir.children.each do |path|
+          if path.pipe?
+            if path.fnmatch("*.r") # Read pipe
+              # debug "Found read pipe: #{path}"
+              @rfile ||= path
+            elsif path.fnmatch("*.w") # Write pipe
+              # debug "Found write pipe: #{path}"
+              @wfile ||= path
+            end
+          else
+            debug "Non-pipe found! #{path}"
+          end
+        end
+        raise ArgumentError, t('no_pipes', :path => pipedir.to_s) if [@rfile, @wfile].any? {|p| p.nil? }
+
+        begin
+          debug "Opening write pipe."
+          @w = open_write_pipe
+          @w.sync = true
+          read_ready = false
+          # Using threads, we get around JRuby's blocking-open
+          # behavior. The main thread pumps the write pipe full of
+          # data so that run_erl will start echoing it into the read
+          # pipe once we have started the open. On non-JVM rubies,
+          # O_NONBLOCK works and we proceed as expected.
+          Thread.new do
+            debug "Opening read pipe."
+            @r = open_read_pipe
+            read_ready = true
+          end
+          Thread.pass
+          @w.print "\n" until read_ready
+          command "ok."
+          debug "Initialized console: #{@r.inspect} #{@w.inspect}"
+        rescue => e
+          debug e.message
+          close
+          raise t('no_pipes', :path => pipedir.to_s) + "[ #{e.message} ]"
+        end
+      end
+
+      # Sends an Erlang command to the console
+      # @param [String] cmd an Erlang expression to send to the node
+      def command(cmd)
+        @mutex.synchronize do
+          begin
+            debug "Sending command #{cmd.inspect}"
+            @w.print "#{cmd}\n"
+            wait_for_erlang_prompt
+          rescue SystemCallError
+            close
+          end
+        end
+      end
+
+      # Detects whether the console connection is still open, that is,
+      # if the node hasn't disconnected from the other side of the
+      # pipe.
+      def open?
+        (@r && !@r.closed?) && (@w && !@w.closed?)
+      end
+
+      # Scans the output of the console until an Erlang shell prompt
+      # is found. Called by {#command} to ensure that the submitted
+      # command succeeds.
+      def wait_for_erlang_prompt
+        wait_for @prompt
+      end
+
+      # Scans the output of the console for the given pattern.
+      # @param [String, Regexp] pattern the pattern to scan for
+      def wait_for(pattern)
+        debug "Scanning for #{pattern.inspect}"
+        @r.expect(pattern)
+      end
+
+      # Closes the console by detaching from the pipes.
+      def close
+        @r.close if @r && !@r.closed?
+        @w.close if @w && !@w.closed?
+        freeze
+      end
+
+      protected
+
+      def debug(msg)
+        $stderr.puts msg if ENV["DEBUG_RIAK_CONSOLE"]
+      end
+
+      def open_write_pipe
+        @wfile.open(File::WRONLY|File::NONBLOCK)
+      end
+
+      def open_read_pipe
+        @rfile.open(File::RDONLY|File::NONBLOCK)
+      end
+    end
+  end
+end

data/lib/riak/node/control.rb

@@ -0,0 +1,207 @@
+require 'riak/node/console'
+require 'riak/util/tcp_socket_extensions'
+
+module Riak
+  class Node
+    # Regexp for parsing riak-admin status output. Takes into account
+    # the minor bug fixed by {https://github.com/basho/riak_kv/pull/134}
+    # and multiline output used when lists of things grow long.
+    STATS_REGEXP = /^([^:\n]+)\s:\s((?:.*)(?:\n\s+[^\n]+)*)/
+
+    # Is the node running?
+    # @return [true, false] If the node is running
+    def started?
+      pinged = ping
+      pinged.strip =~ /pong/ || pinged.strip !~ /Node '[^']+' not responding to pings/
+    end
+
+    # Is the node stopped? (opposite of {#started?}).
+    # @return [true, false] If the node is stopped
+    # @see #started?
+    def stopped?
+      !started?
+    end
+
+    # Starts the node.
+    # @return [String] the output of the 'riak start' command
+    def start
+      res = riak 'start'
+      wait_for_startup
+      res
+    end
+
+    # Stops the node
+    # @return [String] the output of the 'riak stop' command
+    def stop
+      res = riak 'stop'
+      wait_for_shutdown
+      res
+    end
+
+    # Restarts the node
+    # @return [String] the output of the 'riak restart' command
+    def restart
+      riak 'restart'
+    end
+
+    # Reboots the node
+    # @return [String] the output of the 'riak reboot' command
+    def restart
+      riak 'reboot'
+    end
+
+    # Pings the node
+    # @return [String] the output of the 'riak ping' command
+    def ping
+      begin
+        riak 'ping'
+      rescue SystemCallError
+        # If the control script doesn't exist or has the wrong
+        # permissions, we should still return something sane so we can
+        # do the right thing.
+        "Node '#{name}' not responding to pings."
+      end
+    end
+
+    # Attach to the node's console via the pipe.
+    # @return [Riak::Node::Console] A console manager for sending
+    #    commands to the Riak node
+    # @see #with_console
+    def attach
+      Console.open self
+    end
+
+    # Execute the block against the Riak node's console.
+    # @yield [console] A block of commands to be run against the console
+    # @yieldparam [Riak::Node::Console] console A console manager for
+    #    sending commands to the Riak node
+    def with_console
+      begin
+        console = attach
+        yield console
+      ensure
+        console.close if console
+      end
+    end
+
+    # Joins the node to another node to create a cluster.
+    # @return [String] the output of the 'riak-admin join' command
+    def join(node)
+      node = node.name if Node === node
+      riak_admin 'join', node
+    end
+
+    # Removes this node from its current cluster, handing off all data.
+    # @return [String] the output of the 'riak-admin leave' command
+    def leave
+      riak_admin 'leave'
+    end
+
+    # Forcibly removes a node from the current cluster without
+    # invoking handoff.
+    # @return [String] the output of the 'riak-admin remove <node>'
+    #   command
+    def remove(node)
+      node = node.name if Node === node
+      riak_admin 'remove', node
+    end
+
+    # Captures the status of the node.
+    # @return [Hash] a collection of information about the node
+    def status
+      output = riak_admin 'status'
+      if $?.success?
+        result = {}
+        Hash[output.scan(STATS_REGEXP)]
+      end
+    end
+
+    # Detects whether the node's cluster has converged on the ring.
+    # @return [true,false] whether the ring is stable
+    def ringready?
+      output = riak_admin 'ringready'
+      output =~ /^TRUE/ || $?.success?
+    end
+
+    # Lists riak_core applications that have registered as available,
+    # e.g.  ["riak_kv", "riak_search", "riak_pipe"]
+    # @return [Array<String>] a list of running services
+    def services
+      output = riak_admin 'services'
+      if $?.success?
+        output.strip.match(/^\[(.*)\]$/)[1].split(/,/)
+      else
+        []
+      end
+    end
+
+    # Forces the node to restart/reload its JavaScript VMs,
+    # effectively reloading any user-provided code.
+    def js_reload
+      riak_admin 'js_reload'
+    end
+
+    # Provides the status of members of the ring.
+    # @return [Hash] a collection of stats about ring members
+    def member_status
+      output = riak_admin 'member_status'
+      result = {}
+      if $?.success?
+        output.each_line do |line|
+          next if line =~ /^(?:[=-]|Status)+/  # Skip the pretty headers
+          if line =~ %r{^Valid:(\d+) / Leaving:(\d+) / Exiting:(\d+) / Joining:(\d+) / Down:(\d+)}
+            result.merge!(:valid =>   $1.to_i,
+                          :leaving => $2.to_i,
+                          :exiting => $3.to_i,
+                          :joining => $4.to_i,
+                          :down =>    $5.to_i)
+          else
+            result[:members] ||= {}
+            status, ring, pending, node = line.split(/\s+/)
+            node = $1 if node =~ /^'(.*)'$/
+            ring = $1.to_f if ring =~ /(\d+\.\d+)%/
+            result[:members][node] = {
+              :status => status,
+              :ring => ring,
+              :pending => (pending == '--') ? 0 : pending.to_i
+            }
+          end
+        end
+      end
+      result
+    end
+
+    # @return [Array<String>] a list of node names that are also
+    #   members of this node's cluster, and empty list if the
+    #   {#member_status} fails or no other nodes are present
+    def peers
+      all_nodes = member_status[:members] && member_status[:members].keys.reject {|n| n == name }
+      all_nodes || []
+    end
+
+    protected
+    # Runs a command using the 'riak' control script.
+    def riak(*commands)
+      `#{control_script} #{commands.join(' ')} 2>&1`
+    end
+
+    # Runs a command using the 'riak-admin' script.
+    def riak_admin(*commands)
+      `#{admin_script} #{commands.join(' ')} 2>&1`
+    end
+
+    # Waits for the HTTP port to become available, which is a better
+    # indication of readiness than the start script finishing.
+    def wait_for_startup
+      TCPSocket.wait_for_service_with_timeout(:host => http_ip,
+                                              :port => http_port,
+                                              :timeout => 10)
+    end
+
+    def wait_for_shutdown
+      TCPSocket.wait_for_service_termination_with_timeout(:host => http_ip,
+                                                          :port => http_port,
+                                                          :timeout => 10)
+    end
+  end
+end