riak-client 0.9.8 → 1.0.0.beta

data/lib/riak/node/console.rb

@@ -0,0 +1,139 @@
+require 'expect'
+require 'pathname'
+require 'riak/util/translation'
+
+if ENV['DEBUG_RIAK_CONSOLE']
+  $expect_verbose = true
+end
+
+module Riak
+  class Node
+    # Eases working with the Erlang console when attached to the Riak
+    # node.
+    class Console
+      include Util::Translation
+
+      # @return [String] the name of the connected node
+      attr_accessor :nodename
+
+      # Opens a {Console} by connecting to the node.
+      # @return [Console] the opened console
+      def self.open(node)
+        new node.pipe, node.name
+      end
+
+      # Creates a {Console} from the IO pipes connected to the node.
+      # @param [String,Pathname] pipedir path to the pipes opened by
+      #   run_erl
+      # @param [String] nodename the name of the node the Console will
+      #   be attached to
+      def initialize(pipedir, nodename)
+        @nodename = nodename
+        @mutex = Mutex.new
+        @winch = Signal.trap("WINCH", &method(:handle_winch))
+        @prompt = /\(#{Regexp.escape(nodename)}\)\d+>\s*/
+        pipedir = Pathname(pipedir)
+        pipedir.children.each do |path|
+          if path.pipe?
+            if path.fnmatch("*.r") # Read pipe
+              # debug "Found read pipe: #{path}"
+              @rfile ||= path
+            elsif path.fnmatch("*.w") # Write pipe
+              # debug "Found write pipe: #{path}"
+              @wfile ||= path
+            end
+          else
+            debug "Non-pipe found! #{path}"
+          end
+        end
+        raise ArgumentError, t('no_pipes', :path => pipedir.to_s) if [@rfile, @wfile].any? {|p| p.nil? }
+        # We have to open the read pipe AFTER we have sent some data
+        # to the write pipe, otherwise JRuby hangs.
+        begin
+          debug "Opening write pipe."
+          @w = open_write_pipe
+          @w.sync = true
+          debug "Opening read pipe."
+          @r = open_read_pipe
+          command 'ok.'
+          debug "Initialized console: #{@r.inspect} #{@w.inspect}"
+        rescue => e
+          debug e.message
+          close
+          raise t('no_pipes', :path => pipedir.to_s) + "[ #{e.message} ]"
+        end
+      end
+
+      # Sends an Erlang command to the console
+      # @param [String] cmd an Erlang expression to send to the node
+      def command(cmd)
+        @mutex.synchronize do
+          begin
+            debug "Sending command #{cmd.inspect}"
+            @w.print "#{cmd}\n"
+            wait_for_erlang_prompt
+          rescue SystemCallError
+            close
+          end
+        end
+      end
+
+      # Detects whether the console connection is still open, that is,
+      # if the node hasn't disconnected from the other side of the
+      # pipe.
+      def open?
+        (@r && !@r.closed?) && (@w && !@w.closed?)
+      end
+
+      # Scans the output of the console until an Erlang shell prompt
+      # is found. Called by {#command} to ensure that the submitted
+      # command succeeds.
+      def wait_for_erlang_prompt
+        wait_for @prompt
+      end
+
+      # Scans the output of the console for the given pattern.
+      # @param [String, Regexp] pattern the pattern to scan for
+      def wait_for(pattern)
+        debug "Scanning for #{pattern.inspect}"
+        @r.expect(pattern)
+      end
+
+      # Closes the console by detaching from the pipes.
+      def close
+        @r.close if @r && !@r.closed?
+        @w.close if @w && !@w.closed?
+        Signal.trap("WINCH", @winch)
+        freeze
+      end
+
+      protected
+      # Handles the "window change" signal by faking it.
+      def handle_winch
+        debug "WINCHED!"
+        @w.print "\033_winsize=80,26\033\\"
+        Signal.trap("WINCH", &method(:handle_winch))
+      end
+
+      def debug(msg)
+        $stderr.puts msg if ENV["DEBUG_RIAK_CONSOLE"]
+      end
+
+      def open_write_pipe
+        if defined?(::Java)
+          java.io.FileOutputStream.new(@wfile.to_s).to_io
+        else
+          @wfile.open(File::WRONLY|File::NONBLOCK)
+        end
+      end
+
+      def open_read_pipe
+        if defined?(::Java)
+          IO.popen("cat #{@rfile}", "rb")
+        else
+          @rfile.open(File::RDONLY|File::NONBLOCK)
+        end
+      end
+    end
+  end
+end

data/lib/riak/node/control.rb

@@ -0,0 +1,207 @@
+require 'riak/node/console'
+require 'riak/util/tcp_socket_extensions'
+
+module Riak
+  class Node
+    # Regexp for parsing riak-admin status output. Takes into account
+    # the minor bug fixed by {https://github.com/basho/riak_kv/pull/134}
+    # and multiline output used when lists of things grow long.
+    STATS_REGEXP = /^([^:\n]+)\s:\s((?:.*)(?:\n\s+[^\n]+)*)/
+
+    # Is the node running?
+    # @return [true, false] If the node is running
+    def started?
+      pinged = ping
+      pinged.strip =~ /pong/ || pinged.strip !~ /Node '[^']+' not responding to pings/
+    end
+
+    # Is the node stopped? (opposite of {#started?}).
+    # @return [true, false] If the node is stopped
+    # @see #started?
+    def stopped?
+      !started?
+    end
+
+    # Starts the node.
+    # @return [String] the output of the 'riak start' command
+    def start
+      res = riak 'start'
+      wait_for_startup
+      res
+    end
+
+    # Stops the node
+    # @return [String] the output of the 'riak stop' command
+    def stop
+      res = riak 'stop'
+      wait_for_shutdown
+      res
+    end
+
+    # Restarts the node
+    # @return [String] the output of the 'riak restart' command
+    def restart
+      riak 'restart'
+    end
+
+    # Reboots the node
+    # @return [String] the output of the 'riak reboot' command
+    def restart
+      riak 'reboot'
+    end
+
+    # Pings the node
+    # @return [String] the output of the 'riak ping' command
+    def ping
+      begin
+        riak 'ping'
+      rescue SystemCallError
+        # If the control script doesn't exist or has the wrong
+        # permissions, we should still return something sane so we can
+        # do the right thing.
+        "Node '#{name}' not responding to pings."
+      end
+    end
+
+    # Attach to the node's console via the pipe.
+    # @return [Riak::Node::Console] A console manager for sending
+    #    commands to the Riak node
+    # @see #with_console
+    def attach
+      Console.open self
+    end
+
+    # Execute the block against the Riak node's console.
+    # @yield [console] A block of commands to be run against the console
+    # @yieldparam [Riak::Node::Console] console A console manager for
+    #    sending commands to the Riak node
+    def with_console
+      begin
+        console = attach
+        yield console
+      ensure
+        console.close if console
+      end
+    end
+
+    # Joins the node to another node to create a cluster.
+    # @return [String] the output of the 'riak-admin join' command
+    def join(node)
+      node = node.name if Node === node
+      riak_admin 'join', node
+    end
+
+    # Removes this node from its current cluster, handing off all data.
+    # @return [String] the output of the 'riak-admin leave' command
+    def leave
+      riak_admin 'leave'
+    end
+
+    # Forcibly removes a node from the current cluster without
+    # invoking handoff.
+    # @return [String] the output of the 'riak-admin remove <node>'
+    #   command
+    def remove(node)
+      node = node.name if Node === node
+      riak_admin 'remove', node
+    end
+
+    # Captures the status of the node.
+    # @return [Hash] a collection of information about the node
+    def status
+      output = riak_admin 'status'
+      if $?.success?
+        result = {}
+        Hash[output.scan(STATS_REGEXP)]
+      end
+    end
+
+    # Detects whether the node's cluster has converged on the ring.
+    # @return [true,false] whether the ring is stable
+    def ringready?
+      output = riak_admin 'ringready'
+      output =~ /^TRUE/ || $?.success?
+    end
+
+    # Lists riak_core applications that have registered as available,
+    # e.g.  ["riak_kv", "riak_search", "riak_pipe"]
+    # @return [Array<String>] a list of running services
+    def services
+      output = riak_admin 'services'
+      if $?.success?
+        output.strip.match(/^\[(.*)\]$/)[1].split(/,/)
+      else
+        []
+      end
+    end
+
+    # Forces the node to restart/reload its JavaScript VMs,
+    # effectively reloading any user-provided code.
+    def js_reload
+      riak_admin 'js_reload'
+    end
+
+    # Provides the status of members of the ring.
+    # @return [Hash] a collection of stats about ring members
+    def member_status
+      output = riak_admin 'member_status'
+      result = {}
+      if $?.success?
+        output.each_line do |line|
+          next if line =~ /^(?:[=-]|Status)+/  # Skip the pretty headers
+          if line =~ %r{^Valid:(\d+) / Leaving:(\d+) / Exiting:(\d+) / Joining:(\d+) / Down:(\d+)}
+            result.merge!(:valid =>   $1.to_i,
+                          :leaving => $2.to_i,
+                          :exiting => $3.to_i,
+                          :joining => $4.to_i,
+                          :down =>    $5.to_i)
+          else
+            result[:members] ||= {}
+            status, ring, pending, node = line.split(/\s+/)
+            node = $1 if node =~ /^'(.*)'$/
+            ring = $1.to_f if ring =~ /(\d+\.\d+)%/
+            result[:members][node] = {
+              :status => status,
+              :ring => ring,
+              :pending => (pending == '--') ? 0 : pending.to_i
+            }
+          end
+        end
+      end
+      result
+    end
+
+    # @return [Array<String>] a list of node names that are also
+    #   members of this node's cluster, and empty list if the
+    #   {#member_status} fails or no other nodes are present
+    def peers
+      all_nodes = member_status[:members] && member_status[:members].keys.reject {|n| n == name }
+      all_nodes || []
+    end
+
+    protected
+    # Runs a command using the 'riak' control script.
+    def riak(*commands)
+      `#{control_script} #{commands.join(' ')} 2>&1`
+    end
+
+    # Runs a command using the 'riak-admin' script.
+    def riak_admin(*commands)
+      `#{admin_script} #{commands.join(' ')} 2>&1`
+    end
+
+    # Waits for the HTTP port to become available, which is a better
+    # indication of readiness than the start script finishing.
+    def wait_for_startup
+      TCPSocket.wait_for_service_with_timeout(:host => http_ip,
+                                              :port => http_port,
+                                              :timeout => 10)
+    end
+
+    def wait_for_shutdown
+      TCPSocket.wait_for_service_termination_with_timeout(:host => http_ip,
+                                                          :port => http_port,
+                                                          :timeout => 10)
+    end
+  end
+end

data/lib/riak/node/defaults.rb

@@ -0,0 +1,70 @@
+require 'riak/core_ext/deep_dup'
+
+module Riak
+  class Node
+    # Settings based on Riak master/1.0.
+    ENV_DEFAULTS = {
+      :riak_core => {
+        :ring_creation_size => 64
+      },
+      :riak_kv => {
+        :storage_backend => :riak_kv_bitcask_backend,
+        :map_js_vm_count => 8,
+        :reduce_js_vm_count => 6,
+        :hook_js_vm_count => 2,
+        :mapper_batch_size => 5,
+        :js_max_vm_mem => 8,
+        :js_thread_stack => 16,
+        :riak_kv_stat => true,
+        :legacy_stats => false,
+        :vnode_vclocks => true,
+        :http_url_encoding => :on,
+        :legacy_keylisting => false,
+        :mapred_system => :pipe,
+        :add_paths => []
+      },
+      :riak_search => {
+        :enabled => true
+      },
+      :luwak => {
+        :enabled => true
+      },
+      :merge_index => {
+        :buffer_rollover_size => 1048576,
+        :max_compact_segments => 20
+      },
+      :eleveldb => {},
+      :bitcask => {},
+      :lager => {
+        :crash_log_size => 65536,
+        :error_logger_redirect => true
+      },
+      :riak_sysmon => {
+        :process_limit => 30,
+        :port_limit => 30,
+        :gc_ms_limit => 50,
+        :heap_word_limit => 10485760
+      },
+      :sasl => {
+        :sasl_error_logger => false
+      }
+    }.freeze
+
+    # Based on Riak master/1.0.
+    VM_DEFAULTS = {
+      "+K" => true,
+      "+A" => 64,
+      "-smp" => "enable",
+      "+W" => "w",
+      "-env ERL_MAX_PORTS" => 4096,
+      "-env ERL_FULLSWEEP_AFTER" => 0
+    }.freeze
+
+    protected
+    # Populates the defaults
+    def set_defaults
+      @env = ENV_DEFAULTS.deep_dup
+      @vm = VM_DEFAULTS.deep_dup
+    end
+  end
+end

data/lib/riak/node/generation.rb

@@ -0,0 +1,99 @@
+require 'yaml'
+
+module Riak
+  class Node
+    # Does the node exist on disk?
+    def exist?
+      manifest.exist?
+    end
+
+    # Deletes the node and regenerates it.
+    def recreate
+      delete
+      create
+    end
+
+    # Generates the node.
+    def create
+      unless exist?
+        create_directories
+        write_scripts
+        write_vm_args
+        write_app_config
+        write_manifest
+      end
+    end
+
+    # Clears data from known data directories. Stops the node if it is
+    # running.
+    def drop
+      was_started = started?
+      stop if was_started
+      data.children.each {|dir| dir.children.each {|c| c.rmtree } }
+      start if was_started
+    end
+
+    # Removes the node from disk and freezes the object.
+    def destroy
+      delete
+      freeze
+    end
+
+    protected
+    def delete
+      stop unless stopped?
+      root.rmtree if root.exist?
+    end
+
+    def create_directories
+      root.mkpath
+      NODE_DIRECTORIES.each {|d| send(d).mkpath }
+    end
+
+    def write_vm_args
+      (etc + 'vm.args').open('w') do |f|
+        vm.each do |k,v|
+          f.puts "#{k} #{v}"
+        end
+      end
+    end
+
+    def write_app_config
+      (etc + 'app.config').open('w') do |f|
+        f.write to_erlang_config(env) + '.'
+      end
+    end
+
+    def write_scripts
+      [control_script, admin_script].each {|s| write_script(s.basename, s) }
+    end
+
+    def write_script(name, target)
+      source_script = source + name
+      target.open('wb') do |f|
+        source_script.readlines.each do |line|
+          line.sub!(/(RUNNER_SCRIPT_DIR=)(.*)/, '\1' + bin.to_s)
+          line.sub!(/(RUNNER_ETC_DIR=)(.*)/, '\1' + etc.to_s)
+          line.sub!(/(RUNNER_USER=)(.*)/, '\1')
+          line.sub!(/(RUNNER_LOG_DIR=)(.*)/, '\1' + log.to_s)
+          line.sub!(/(PIPE_DIR=)(.*)/, '\1' + pipe.to_s)
+          line.sub!('grep "$RUNNER_BASE_DIR/.*/[b]eam"', 'grep "$RUNNER_ETC_DIR/app.config"')
+          if line.strip == "RUNNER_BASE_DIR=${RUNNER_SCRIPT_DIR%/*}"
+            line = "RUNNER_BASE_DIR=#{source.parent.to_s}\n"
+          end
+          f.write line
+        end
+      end
+      target.chmod 0755
+    end
+
+    def write_manifest
+      # TODO: For now this only saves the information that was used when
+      # configuring the node. Later we'll verify/warn if the settings
+      # used differ on subsequent generations.
+      @configuration[:env] = @env
+      @configuration[:vm] = @vm
+      manifest.open('w') {|f| YAML.dump(@configuration, f) }
+    end
+  end
+end