riak-client 1.0.0.beta → 1.0.0

data/lib/riak/client/pool.rb

@@ -0,0 +1,180 @@
+require 'thread'
+
+module Riak
+  class Client
+    # A re-entrant thread-safe resource pool. Generates new resources on
+    # demand.
+    # @private
+    class Pool
+      # Raised when a taken element should be deleted from the pool.
+      class BadResource < RuntimeError
+      end
+
+      # An element of the pool. Comprises an object with an owning
+      # thread.
+      # @private
+      class Element
+        attr_accessor :object
+        attr_accessor :owner
+        def initialize(object)
+          @object = object
+          @owner = owner
+        end
+
+        # Claims this element of the pool for the current Thread.
+        def lock
+          self.owner = Thread.current
+        end
+
+        # Is this element locked/claimed?
+        def locked?
+          not owner.nil?
+        end
+
+        # Releases this element of the pool from the current Thread.
+        def unlock
+          self.owner = nil
+        end
+
+        # Is this element available for use?
+        def unlocked?
+          owner.nil?
+        end
+      end
+
+      attr_accessor :pool
+      attr_accessor :open
+      attr_accessor :close
+
+      # Open is a callable which returns a new object for the pool. Close is
+      # called with an object before it is freed.
+      def initialize(open, close)
+        @open = open
+        @close = close
+        @lock = Mutex.new
+        @iterator = Mutex.new
+        @element_released = ConditionVariable.new
+        @pool = Set.new
+      end
+
+      # On each element of the pool, calls close(element) and removes it.
+      # @private
+      def clear
+        each_element do |e|
+          delete_element e
+        end
+      end
+      alias :close :clear
+
+      # Deletes an element of the pool. Calls close on its object.
+      # Not intendend for external use.
+      def delete_element(e)
+        @close.call(e.object)
+        @lock.synchronize do
+          @pool.delete e
+        end
+      end
+
+      # Locks each element in turn and closes/deletes elements for which the
+      # object passes the block.
+      def delete_if
+        raise ArgumentError, "block required" unless block_given?
+
+        each_element do |e|
+          if yield e.object
+            delete_element e
+          end
+        end
+      end
+
+      # Acquire an element of the pool. Yields the object. If all
+      # elements are claimed, it will create another one.
+      # @yield [obj] a block that will perform some action with the
+      #   element of the pool
+      # @yieldparam [Object] resource a resource managed by the pool.
+      #   Locked for the duration of the block
+      # @param [callable] :filter a callable which receives objects and has
+      #   the opportunity to reject each in turn.
+      # @param [Object] :default if no resources are available, use this object
+      #   instead of calling #open.
+      # @private
+      def take(opts = {})
+        unless block_given?
+          raise ArgumentError, "block required"
+        end
+
+        r = nil
+        begin
+          e = nil
+          @lock.synchronize do
+            # Find an existing element.
+            if f = opts[:filter]
+              e = pool.find { |e| e.unlocked? and f.call(e.object) }
+            else
+              e = pool.find { |e| e.unlocked? }
+            end
+
+            unless e
+              # No objects were acceptable
+              resource = opts[:default] || @open.call
+              e = Element.new(resource)
+              pool << e
+            end
+            e.lock
+          end
+
+          r = yield e.object
+        rescue BadResource
+          delete_element e
+          raise
+        ensure
+          # Unlock
+          if e
+            e.unlock
+            @element_released.signal
+          end
+        end
+        r
+      end
+      alias >> take
+
+      # Iterate over a snapshot of the pool. Yielded objects are locked for the
+      # duration of the block. This may block the current thread until elements
+      # are released by other threads.
+      # @private
+      def each_element
+        targets = @pool.to_a
+        unlocked = []
+
+        @iterator.synchronize do
+          until targets.empty?
+            @lock.synchronize do
+              unlocked, targets = targets.partition {|e| e.unlocked? }
+              unlocked.each {|e| e.lock }
+            end
+
+            unlocked.each do |e|
+              begin
+                yield e
+              ensure
+                e.unlock
+              end
+            end
+            @element_released.wait(@iterator) unless targets.empty?
+          end
+        end
+      end
+
+      # As each_element, but yields objects, not wrapper elements.
+      def each
+        each_element do |e|
+          yield e.object
+        end
+      end
+
+      def size
+        @lock.synchronize { @pool.size }
+      end
+    end
+  end
+end

data/lib/riak/client/protobuffs_backend.rb

@@ -8,6 +8,7 @@ module Riak
   class Client
     class ProtobuffsBackend
       include Util::Translation
+      include Util::Escape
 
       # Message Codes Enum
       MESSAGE_CODES = %W[
@@ -46,8 +47,10 @@ module Riak
       end
 
       attr_accessor :client
-      def initialize(client)
+      attr_accessor :node
+      def initialize(client, node)
         @client = client
+        @node = node
       end
 
       simple :ping,          :PingReq
@@ -67,6 +70,11 @@ module Riak
                reduce(%w[riak_kv_mapreduce reduce_identity], :arg => {:reduce_phase_only_1 => true}, :keep => true)).map {|p| p.last }
       end
 
+      # Gracefully shuts down this connection.
+      def teardown
+        reset_socket
+      end
+
       private
       # Implemented by subclasses
       def decode_response
@@ -74,18 +82,20 @@ module Riak
       end
 
       def socket
-        Thread.current[:riakpbc_socket] ||= new_socket
+        @socket ||= new_socket
       end
 
       def new_socket
-        socket = TCPSocket.new(@client.host, @client.pb_port)
+        socket = TCPSocket.new(@node.host, @node.pb_port)
         socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, true)
+        #TODO: Should we set the client ID here?
+        # set_client_id @client.client_id
         socket
       end
 
       def reset_socket
-        socket.close if !socket.closed?
-        Thread.current[:riakpbc_socket] = nil
+        @socket.close if @socket && !@socket.closed?
+        @socket = nil
       end
 
       UINTMAX = 0xffffffff

data/lib/riak/client/search.rb

@@ -20,7 +20,9 @@ module Riak
     def search(*args)
       options = args.extract_options!
       index, query = args[-2], args[-1]  # Allows nil index, while keeping it as firstargument
-      http.search(index, query, options)
+      http do |h|
+        h.search(index, query, options)
+      end
     end
     alias :select :search
 
@@ -46,7 +48,9 @@ module Riak
           end
         end
       end
-      http.update_search_index(index, xml.target!)
+      http do |h|
+        h.update_search_index(index, xml.target!)
+      end
       true
     end
     alias :add_doc :index
@@ -76,7 +80,9 @@ module Riak
           end
         end
       end
-      http.update_search_index(index, xml.target!)
+      http do |h|
+        h.update_search_index(index, xml.target!)
+      end
       true
     end
     alias :delete_doc :remove

data/lib/riak/link.rb

@@ -53,14 +53,12 @@ module Riak
 
     # @return [String] the URL (relative or absolute) of the related resource
     def url(new_scheme=false)
-      if @bucket
-        if new_scheme
-          "/buckets/#{escape(bucket)}" + (key.blank? ? "" : "/keys/#{escape(key)}")
-        else
-          "/riak/#{escape(bucket)}" + (key.blank? ? "" : "/#{escape(key)}")
-        end
+      return @url unless @bucket
+
+      if new_scheme
+        "/buckets/#{escape(bucket)}" + (key.blank? ? "" : "/keys/#{escape(key)}")
       else
-        @url
+        "/riak/#{escape(bucket)}" + (key.blank? ? "" : "/#{escape(key)}")
       end
     end
 

data/lib/riak/locale/en.yml

@@ -30,6 +30,7 @@ en:
     loading_bucket: "while loading bucket '%{name}'"    
     list_buckets: "Riak::Client#buckets is an expensive operation that should not be used in production.\n    %{backtrace}"
     list_keys: "Riak::Bucket#keys is an expensive operation that should not be used in production.\n    %{backtrace}"
+    luwak_unsupported: "Riak server does not support Luwak. Enable it in app.config before using."
     missing_block: "A block must be given."
     missing_host_and_port: "You must specify a host and port, or use the defaults of 127.0.0.1:8098"
     module_function_pair_required: "function must have two elements when an array"

data/lib/riak/node/configuration.rb

@@ -156,6 +156,7 @@ module Riak
       env[:eleveldb][:data_root] ||= (data + 'leveldb').expand_path.to_s
       env[:merge_index][:data_root] ||= (data + 'merge_index').expand_path.to_s
       env[:riak_core][:ring_state_dir] ||= ring.expand_path.to_s
+      env[:riak_core][:slide_private_dir] ||= (data + 'slide-data').expand_path.to_s
       NODE_DIRECTORIES.each do |dir|
         next if [:ring, :pipe].include?(dir)
         env[:riak_core][:"platform_#{dir}_dir"] ||= send(dir).to_s

data/lib/riak/node/defaults.rb

@@ -2,7 +2,7 @@ require 'riak/core_ext/deep_dup'
 
 module Riak
   class Node
-    # Settings based on Riak master/1.0.
+    # Settings based on Riak 1.1.
     ENV_DEFAULTS = {
       :riak_core => {
         :ring_creation_size => 64
@@ -16,11 +16,13 @@ module Riak
         :js_max_vm_mem => 8,
         :js_thread_stack => 16,
         :riak_kv_stat => true,
-        :legacy_stats => false,
+        :legacy_stats => true,
         :vnode_vclocks => true,
         :http_url_encoding => :on,
         :legacy_keylisting => false,
         :mapred_system => :pipe,
+        :mapred_2i_pipe => true,
+        :listkeys_backpressure => true,
         :add_paths => []
       },
       :riak_search => {
@@ -36,21 +38,32 @@ module Riak
       :eleveldb => {},
       :bitcask => {},
       :lager => {
-        :crash_log_size => 65536,
+        :crash_log_size => 10485760,
+        :crash_log_msg_size => 65536,
+        :crash_log_date => "$D0",
+        :crash_log_count => 5,
         :error_logger_redirect => true
       },
       :riak_sysmon => {
         :process_limit => 30,
         :port_limit => 30,
-        :gc_ms_limit => 50,
-        :heap_word_limit => 10485760
+        :gc_ms_limit => 100,
+        :heap_word_limit => 40111000,
+        :busy_port => true,
+        :busy_dist_port => true
       },
       :sasl => {
         :sasl_error_logger => false
+      },
+      :riak_control => {
+        :enabled => false,
+        :auth => :userlist,
+        :userlist => {"user" => "pass"},
+        :admin => true
       }
     }.freeze
 
-    # Based on Riak master/1.0.
+    # Based on Riak 1.1.
     VM_DEFAULTS = {
       "+K" => true,
       "+A" => 64,

data/lib/riak/node/generation.rb

@@ -29,7 +29,13 @@ module Riak
     def drop
       was_started = started?
       stop if was_started
-      data.children.each {|dir| dir.children.each {|c| c.rmtree } }
+      data.children.each do |item|
+        if item.directory?
+          item.children.each {|c| c.rmtree }
+        else
+          item.delete
+        end
+      end
       start if was_started
     end
 
@@ -76,7 +82,8 @@ module Riak
           line.sub!(/(RUNNER_ETC_DIR=)(.*)/, '\1' + etc.to_s)
           line.sub!(/(RUNNER_USER=)(.*)/, '\1')
           line.sub!(/(RUNNER_LOG_DIR=)(.*)/, '\1' + log.to_s)
-          line.sub!(/(PIPE_DIR=)(.*)/, '\1' + pipe.to_s)
+          line.sub!(/(PIPE_DIR=)(.*)/, '\1' + pipe.to_s + "/") # PIPE_DIR must have a trailing slash
+          line.sub!(/(PLATFORM_DATA_DIR=)(.*)/, '\1' + data.to_s)
           line.sub!('grep "$RUNNER_BASE_DIR/.*/[b]eam"', 'grep "$RUNNER_ETC_DIR/app.config"')
           if line.strip == "RUNNER_BASE_DIR=${RUNNER_SCRIPT_DIR%/*}"
             line = "RUNNER_BASE_DIR=#{source.parent.to_s}\n"

data/lib/riak/node/log.rb

@@ -23,8 +23,8 @@ module Riak
     def expand_log_level(level)
       case level
       when Range
-        first = LAGER_LEVELS.index(level.begin) || 0
-        last = LAGER_LEVELS.index(level.end) || -1
+        first = LAGER_LEVELS.index(level.begin.to_sym) || 0
+        last = LAGER_LEVELS.index(level.end.to_sym) || -1
         LAGER_LEVELS[first..last]
       when Symbol
         level

data/lib/riak/node/version.rb

@@ -2,10 +2,18 @@ require 'pathname'
 
 module Riak
   class Node
+    # @return [String] the version of the Riak node
     def version
       @version ||= configure_version
     end
 
+    # @return [Pathname] the location of Riak installation, aka RUNNER_BASE_DIR
+    def base_dir
+      @base_dir ||= configure_base_dir
+    end
+
+    protected
+    # Detects the Riak version from the generated release
     def configure_version
       if base_dir
         versions = (base_dir + 'releases' + 'start_erl.data').read
@@ -13,24 +21,22 @@ module Riak
       end
     end
 
-    def base_dir
-      @base_dir ||= configure_base_dir
-    end
-
+    # Determines the base_dir from source control script
     def configure_base_dir
-      pattern = /^RUNNER_BASE_DIR=(.*)/
-      lines = control_script.readlines.grep(pattern)
-      if lines.empty?
-        nil
+      # Use the script from the source directory so we don't require
+      # it to be generated first.
+      (source + control_script_name).each_line.find {|l| l =~ /^RUNNER_BASE_DIR=(.*)/ }
+
+      # There should only be one matching line, so the contents of $1
+      # will be the matched path. If there's nothing matched, we
+      # return nil.
+      case $1
+      when '${RUNNER_SCRIPT_DIR%/*}'
+        source.parent
+      when String
+        Pathname.new($1).expand_path
       else
-        line = lines.first
-        case line
-        when /^RUNNER_BASE_DIR=$\{RUNNER_SCRIPT_DIR%\/*\}/
-          source.parent
-        else
-          path = pattern.match(line)[1]
-          Pathname.new(path).expand_path if File.directory?(path)
-        end
+        nil
       end
     end
   end