better-riak-client 1.0.5

data/lib/riak/search.rb

@@ -0,0 +1,3 @@
+require 'riak'
+
+warn Riak.t('deprecated.search', :backtrace => "    "+caller.join("\n    "))

data/lib/riak/serializers.rb

@@ -0,0 +1,74 @@
+module Riak
+  module Serializers
+    include Util::Translation
+    extend self
+
+    def [](content_type)
+      serializers[content_type]
+    end
+
+    def []=(content_type, serializer)
+      serializers[content_type] = serializer
+    end
+
+    def serialize(content_type, content)
+      serializer_for(content_type).dump(content)
+    end
+
+    def deserialize(content_type, content)
+      serializer_for(content_type).load(content)
+    end
+
+    private
+
+    def serializer_for(content_type)
+      serializers.fetch(content_type[/^[^;\s]+/]) do
+        raise NotImplementedError.new(t('serializer_not_implemented', :content_type => content_type.inspect))
+      end
+    end
+
+    def serializers
+      @serializers ||= {}
+    end
+
+    module TextPlain
+      extend self
+
+      def dump(object)
+        object.to_s
+      end
+
+      def load(string)
+        string
+      end
+    end
+
+    module ApplicationJSON
+      extend self
+
+      def dump(object)
+        object.to_json(Riak.json_options)
+      end
+
+      def load(string)
+        Riak::JSON.parse(string)
+      end
+    end
+
+    Serializers['text/plain'] = TextPlain
+    Serializers['application/json'] = ApplicationJSON
+    Serializers['application/x-ruby-marshal'] = ::Marshal
+
+    YAML_MIME_TYPES = %w[
+      text/yaml
+      text/x-yaml
+      application/yaml
+      application/x-yaml
+    ]
+
+    YAML_MIME_TYPES.each do |mime_type|
+      Serializers[mime_type] = ::YAML
+    end
+  end
+end
+

data/lib/riak/stamp.rb

@@ -0,0 +1,77 @@
+require 'riak/client'
+require 'riak/util/translation'
+require 'thread'
+
+module Riak
+  # Implements a client-side form of monotonically-increasing k-sorted
+  # unique identifiers.  These are useful for key generation if your
+  # data is time-sequential and needs to be sorted by key, perhaps in
+  # Riak Search. Inspired by Twitter's Snowflake project.
+  class Stamp
+    attr_reader :client
+
+    CLIENT_ID_MASK = (1 << 10) - 1
+    SEQUENCE_MASK = (1 << 12) - 1
+    TIMESTAMP_MASK = (1 << 41) - 1
+    SEQUENCE_SHIFT = 10
+    TIMESTAMP_SHIFT = 22
+
+    # @param [Client] client a {Riak::Client} which will be used for
+    #   the "worker ID" component of the stamp.
+    # @see Client#stamp
+    def initialize(client)
+      @client = client
+      @mutex = Mutex.new
+      @timestamp = time_gen
+      @sequence = 0
+    end
+
+    # Generates a k-sorted unique ID for use as a key or other
+    # disambiguation purposes.
+    def next
+      @mutex.synchronize do
+        now = time_gen
+        if @timestamp == now
+          @sequence = (@sequence + 1) & SEQUENCE_MASK
+          now = wait_for_next_ms(@timestamp) if @sequence == 0
+        else
+          @sequence = 0
+        end
+
+        raise BackwardsClockError.new(@timestamp - now) if now < @timestamp
+
+        @timestamp = now
+        @timestamp << TIMESTAMP_SHIFT | @sequence << SEQUENCE_SHIFT | client_id
+      end
+    end
+
+    private
+    def client_id
+      case id = @client.client_id
+      when Integer
+        id & CLIENT_ID_MASK
+      else
+        id.hash & CLIENT_ID_MASK
+      end
+    end
+
+    def time_gen
+      (Time.now.to_f * 1000).floor & TIMESTAMP_MASK
+    end
+
+    def wait_for_next_ms(start)
+      now = time_gen
+      now = time_gen while now <= start
+      now
+    end
+  end
+
+  # Raised when calling {Stamp#next} and NTP or some other external
+  # event has moved the system clock backwards.
+  class BackwardsClockError < StandardError
+    include Util::Translation
+    def initialize(delay)
+      super t('backwards_clock', :delay => delay)
+    end
+  end
+end

data/lib/riak/test_server.rb

@@ -0,0 +1,89 @@
+require 'riak/node'
+
+if ENV['DEBUG_RIAK_TEST_SERVER']
+  $expect_verbose = true
+end
+
+module Riak
+  # The TestServer is a special {Node} that uses in-memory storage
+  # engines that are easily cleared. This is helpful when running test
+  # suites that store and retrieve objects from Riak and expect a
+  # clean-slate at the beginning of each test. Like {Node}, creation
+  # is idempotent, so you can keep the server around between runs of
+  # your test suite.
+  class TestServer < Node
+    # Creates a TestServer node, using in-memory backends for KV and Search.
+    def initialize(configuration = {})
+      configuration[:env] ||= {}
+      configuration[:env][:riak_kv] ||= {}
+      (configuration[:env][:riak_kv][:add_paths] ||= []) << File.expand_path("../../../erl_src", __FILE__)
+      configuration[:env][:riak_kv][:test] = true
+      configuration[:env][:memory_backend] ||={}
+      configuration[:env][:memory_backend][:test] = true
+      configuration[:env][:riak_search] ||= {}
+      configuration[:env][:riak_search][:search_backend] = :riak_search_test_backend
+      super configuration
+    end
+
+    # Overrides the default {Node#started?} to simply return true if the
+    # console is still attached.
+    def started?
+      open? || super
+    end
+
+    # Overrides the default {Node#start} to return early if the
+    # console is still attached. Otherwise, starts and immediately
+    # attaches the console.
+    def start
+      unless open?
+        super
+        maybe_attach
+      end
+    end
+
+    # Overrides the default {Node#stop} to close the console before
+    # stopping the node.
+    def stop
+      @console.close if @console && !@console.frozen?
+      @console = nil
+      super
+    end
+
+    # Overrides the default {Node#drop} to simply clear the in-memory
+    # backends.
+    def drop
+      begin
+        maybe_attach
+        @console.command "#{kv_backend}:reset()."
+        @console.command "riak_search_test_backend:reset()."
+      rescue IOError
+        retry
+      end
+    end
+
+    protected
+    # Tries to reattach the console if it's closed
+    def maybe_attach
+      unless open?
+        @console.close if @console && !@console.frozen?
+        @console = attach
+      end
+    end
+
+    def open?
+      @console && @console.open?
+    end
+
+    def configure_data
+      super
+      if version < "1.0.0"
+        env[:riak_kv][:storage_backend] = :riak_kv_test014_backend
+      elsif version =~ /^1\.[01]\.\d+$/ # 1.0 and 1.1 series
+        env[:riak_kv][:storage_backend] = :riak_kv_test_backend
+      else
+        # TODO: change this when 1.2+ is released, if it includes riak_kv#314
+        env[:riak_kv][:storage_backend] = :riak_kv_memory_backend
+      end
+    end
+  end
+end

data/lib/riak/util/escape.rb

@@ -0,0 +1,76 @@
+require 'cgi'
+require 'uri'
+
+module Riak
+  class << self
+    # @see #escaper=
+    attr_reader :escaper
+
+    # Sets the class used for escaping URLs (buckets and keys) sent to
+    # Riak. Currently only supports URI and CGI, and defaults to URI.
+    # @param [Symbol,String,Class] esc A representation of which
+    #   escaping class to use, either the Class itself or a String or
+    #   Symbol name
+    # @see Riak::Util::Escape
+    def escaper=(esc)
+      case esc
+      when Symbol, String
+        @escaper = ::Object.const_get(esc.to_s.upcase.intern) if esc.to_s =~ /uri|cgi/i
+      when Class, Module
+        @escaper = esc
+      end
+    end
+
+    # In Riak 1.0+, buckets and keys are decoded internally before
+    # being stored. This increases compatibility with the Protocol
+    # Buffers transport and reduces inconsistency of link-walking
+    # vs. regular operations. If the node you are connecting to has
+    # set {http_url_encoding, on}, set this to true. Default is false.
+    # @return [true,false] Whether Riak decodes URL-encoded paths and headers
+    attr_accessor :url_decoding
+  end
+
+  self.escaper = URI
+  self.url_decoding = false
+
+  module Util
+    # Methods for escaping URL segments.
+    module Escape
+      # Conditionally escapes buckets and keys depending on whether
+      # Riak is configured to decode them. This is used in situations
+      # where the bucket or key is not part of a URL, but would need
+      # to be escaped on Riak 0.14 and earlier so that the name
+      # matches.
+      # @param [String] bucket_or_key the bucket or key name
+      # @return [String] the escaped path segment
+      def maybe_escape(bucket_or_key)
+        Riak.url_decoding ? bucket_or_key : escape(bucket_or_key)
+      end
+
+      # Escapes bucket or key names that may contain slashes for use in URLs.
+      # @param [String] bucket_or_key the bucket or key name
+      # @return [String] the escaped path segment
+      def escape(bucket_or_key)
+        Riak.escaper.escape(bucket_or_key.to_s).gsub("+", "%20").gsub('/', "%2F")
+      end
+
+      # Conditionally unescapes buckets and keys depending on whether
+      # Riak is configured to decode them. This is used in situations
+      # where the bucket or key is not part of a URL, but would need
+      # to be escaped on Riak 0.14 and earlier so that the name
+      # matches.
+      # @param [String] bucket_or_key the escaped bucket or key name
+      # @return [String] the unescaped path segment
+      def maybe_unescape(bucket_or_key)
+        Riak.url_decoding ? bucket_or_key : unescape(bucket_or_key)
+      end
+
+      # Unescapes bucket or key names in URLs.
+      # @param [String] bucket_or_key the bucket or key name
+      # @return [String] the unescaped name
+      def unescape(bucket_or_key)
+        Riak.escaper.unescape(bucket_or_key.to_s)
+      end
+    end
+  end
+end

data/lib/riak/util/headers.rb

@@ -0,0 +1,53 @@
+require 'net/http'
+
+# Splits headers into < 8KB chunks
+# @private
+module Net::HTTPHeader
+  def each_capitalized
+    # 1.9 check
+    respond_to?(:enum_for) and (block_given? or return enum_for(__method__))
+    @header.each do |k,v|
+      base_length = "#{k}: \r\n".length
+      values = v.map {|i| i.to_s.split(", ") }.flatten
+      while !values.empty?
+        current_line = ""
+        while values.first && current_line.length + base_length + values.first.length + 2 < 8192
+          val = values.shift.strip
+          current_line += current_line.empty? ? val : ", #{val}"
+        end
+        yield capitalize(k), current_line
+      end
+    end
+  end
+end
+
+module Riak
+  module Util
+    # Represents headers from an HTTP request or response.
+    # Used internally by HTTP backends for processing headers.
+    class Headers
+      include Net::HTTPHeader
+
+      def initialize
+        initialize_http_header({})
+      end
+
+      # Parse a single header line into its key and value
+      # @param [String] chunk a single header line
+      def self.parse(chunk)
+        line = chunk.strip
+        # thanks Net::HTTPResponse
+        return [nil,nil] if chunk =~ /\AHTTP(?:\/(\d+\.\d+))?\s+(\d\d\d)\s*(.*)\z/in
+        m = /\A([^:]+):\s*/.match(line)
+        [m[1], m.post_match] rescue [nil, nil]
+      end
+
+      # Parses a header line and adds it to the header collection
+      # @param [String] chunk a single header line
+      def parse(chunk)
+        key, value = self.class.parse(chunk)
+        add_field(key, value) if key && value
+      end
+    end
+  end
+end

data/lib/riak/util/multipart/stream_parser.rb

@@ -0,0 +1,62 @@
+require 'riak/util/translation'
+require 'riak/util/multipart'
+
+module Riak
+  module Util
+    module Multipart
+      # This class is parses chunked/streamed multipart HTTP
+      # streams. It is used by streaming MapReduce queries, and in the
+      # future streaming key-lists (once implemented on the Riak side).
+      class StreamParser
+        include Multipart
+        include Translation
+        # Creates a new StreamParser.
+        #
+        # Example usage:
+        #     http.get(200, "/riak", "foo", {}, &StreamParser.new {|part| ... })
+        #
+        # @yield [Hash] parts of the multipart/mixed stream,
+        #               containing :headers and :body keys
+        def initialize(&block)
+          raise ArgumentError, t('missing_block') unless block_given?
+          @buffer = ""
+          @block = block
+          @state = :get_boundary
+        end
+
+        # Accepts a chunk of the HTTP response stream, and yields to
+        # the block when appropriate.
+        def accept(chunk)
+          @buffer << chunk
+          @state = send(@state)
+        end
+
+        # Returns a Proc that can be passed to an HTTP request method.
+        def to_proc
+          method(:accept).to_proc
+        end
+
+        private
+        CAPTURE_BOUNDARY = /^--([A-Za-z0-9\'()+_,-.\/:=?]+)\r?\n/
+
+        def get_boundary
+          if @buffer =~ CAPTURE_BOUNDARY
+            @re = /\r?\n--#{Regexp.escape($1)}(?:--)?\r?\n/
+            @buffer = $~.post_match
+            buffering
+          else
+            :get_boundary
+          end
+        end
+
+        def buffering
+          while @buffer =~ @re
+            @block.call parse_multipart_section($~.pre_match)
+            @buffer = $~.post_match
+          end
+          :buffering
+        end
+      end
+    end
+  end
+end

data/lib/riak/util/multipart.rb

@@ -0,0 +1,52 @@
+require 'riak/util/headers'
+
+module Riak
+  module Util
+    # Utility methods for handling multipart/mixed responses
+    module Multipart
+      extend self
+      # Parses a multipart/mixed body into its constituent parts, including nested multipart/mixed sections
+      # @param [String] data the multipart body data
+      # @param [String] boundary the boundary string given in the Content-Type header
+      def parse(data, boundary)
+        contents = data.match(end_boundary_regex(boundary)).pre_match rescue ""
+        contents.split(inner_boundary_regex(boundary)).reject(&:blank?).map do |part|
+          parse_multipart_section(part)
+        end.compact
+      end
+
+      # Extracts the boundary string from a Content-Type header that is a multipart type
+      # @param [String] header_string the Content-Type header
+      # @return [String] the boundary string separating each part
+      def extract_boundary(header_string)
+        $1 if header_string =~ /boundary=([A-Za-z0-9\'()+_,-.\/:=?]+)/
+      end
+
+      private
+      def end_boundary_regex(boundary)
+        /\r?\n--#{Regexp.escape(boundary)}--\r?\n/
+      end
+
+      def inner_boundary_regex(boundary)
+        /\r?\n--#{Regexp.escape(boundary)}\r?\n/
+      end
+
+      def parse_multipart_section(part)
+        headers = Headers.new
+        if md = part.match(/\r?\n\r?\n/)
+          body = md.post_match
+          md.pre_match.split(/\r?\n/).each do |line|
+            headers.parse(line)
+          end
+
+          if headers["content-type"] =~ /multipart\/mixed/
+            boundary = extract_boundary(headers.to_hash["content-type"].first)
+            parse(body, boundary)
+          else
+            {:headers => headers.to_hash, :body => body}
+          end
+        end
+      end
+    end
+  end
+end