riak-client-noenc 1.0.0

data/lib/riak/map_reduce/filter_builder.rb

@@ -0,0 +1,103 @@
+
+require 'riak/util/translation'
+
+module Riak
+  class MapReduce
+    # Builds key-filter lists for MapReduce inputs in a DSL-like fashion.
+    class FilterBuilder
+      include Util::Translation
+
+      # Known filters available in riak_kv_mapred_filters, mapped to
+      # their arities. These are turned into instance methods.
+      # Example:
+      #
+      #    FilterBuilder.new do
+      #      string_to_int
+      #      less_than 50
+      #    end
+      FILTERS = {
+        :int_to_string => 0,
+        :string_to_int => 0,
+        :float_to_string => 0,
+        :string_to_float => 0,
+        :to_upper => 0,
+        :to_lower => 0,
+        :tokenize => 2,
+        :urldecode => 0,
+        :greater_than => 1,
+        :less_than => 1,
+        :greater_than_eq => 1,
+        :less_than_eq => 1,
+        :between => [2, 3],
+        :matches => 1,
+        :neq => 1,
+        :eq => 1,
+        :set_member => -1,
+        :similar_to => 2,
+        :starts_with => 1,
+        :ends_with => 1
+      }
+
+      # Available logical operations for joining filter chains. These
+      # are turned into instance methods with leading underscores,
+      # with aliases to uppercase versions.
+      # Example:
+      #
+      #    FilterBuilder.new do
+      #      string_to_int
+      #      AND do
+      #        greater_than_eq 50
+      #        neq 100
+      #      end
+      #    end
+      LOGICAL_OPERATIONS = %w{and or not}
+
+      FILTERS.each do |f, arity|
+        arities = [arity].flatten
+
+        define_method(f) { |*args|
+          unless arities.include?(-1) or arities.include?(args.size)
+            raise ArgumentError.new t("filter_arity_mismatch",
+              :filter => f,
+              :expected => arities,
+              :received => args.size
+            )
+          end
+
+          @filters << [f, *args]
+        }
+      end
+
+      LOGICAL_OPERATIONS.each do |op|
+        # NB: string eval is needed here because in ruby 1.8, blocks can't yield to
+        # other blocks
+        class_eval <<-CODE
+          def _#{op}(&block)
+            raise ArgumentError.new(t('filter_needs_block', :filter => '#{op}')) unless block_given?
+            @filters << [:#{op}, self.class.new(&block).to_a]
+          end
+          alias :#{op.to_s.upcase} :_#{op}
+        CODE
+      end
+
+      # Creates a new FilterBuilder. Pass a block that will be
+      # instance_eval'ed to construct the sequence of filters.
+      def initialize(&block)
+        @filters = []
+        instance_eval(&block) if block_given?
+      end
+
+      # Wraps multi-step filters for use inside logical
+      # operations. Does not correspond to an actual filter.
+      def sequence(&block)
+        @filters << self.class.new(&block).to_a
+      end
+      alias :seq :sequence
+
+      # @return A list of filters for handing to the MapReduce inputs.
+      def to_a
+        @filters
+      end
+    end
+  end
+end

data/lib/riak/map_reduce/phase.rb

@@ -0,0 +1,98 @@
+
+require 'riak/json'
+require 'riak/util/translation'
+require 'riak/walk_spec'
+
+module Riak
+  class MapReduce
+    # Represents an individual phase in a map-reduce pipeline. Generally you'll want to call
+    # methods of MapReduce instead of using this directly.
+    class Phase
+      include Util::Translation
+      # @return [Symbol] the type of phase - :map, :reduce, or :link
+      attr_accessor :type
+
+      # @return [String, Array<String, String>, Hash, WalkSpec] For :map and :reduce types, the Javascript function to run (as a string or hash with bucket/key), or the module + function in Erlang to run. For a :link type, a {Riak::WalkSpec} or an equivalent hash.
+      attr_accessor :function
+
+      # @return [String] the language of the phase's function - "javascript" or "erlang". Meaningless for :link type phases.
+      attr_accessor :language
+
+      # @return [Boolean] whether results of this phase will be returned
+      attr_accessor :keep
+
+      # @return [Array] any extra static arguments to pass to the phase
+      attr_accessor :arg
+
+      # Creates a phase in the map-reduce pipeline
+      # @param [Hash] options options for the phase
+      # @option options [Symbol] :type one of :map, :reduce, :link
+      # @option options [String] :language ("javascript") "erlang" or "javascript"
+      # @option options [String, Array, Hash] :function In the case of Javascript, a literal function in a string, or a hash with :bucket and :key. In the case of Erlang, an Array of [module, function].  For a :link phase, a hash including any of :bucket, :tag or a WalkSpec.
+      # @option options [Boolean] :keep (false) whether to return the results of this phase
+      # @option options [Array] :arg (nil) any extra static arguments to pass to the phase
+      def initialize(options = {})
+        self.type = options[:type]
+        self.language = options[:language] || "javascript"
+        self.function = options[:function]
+        self.keep = options[:keep] || false
+        self.arg = options[:arg]
+      end
+
+      def type=(value)
+        raise ArgumentError, t("invalid_phase_type") unless value.to_s =~ /^(map|reduce|link)$/i
+        @type = value.to_s.downcase.to_sym
+      end
+
+      def function=(value)
+        case value
+        when Array
+          raise ArgumentError, t("module_function_pair_required") unless value.size == 2
+          @language = "erlang"
+        when Hash
+          raise ArgumentError, t("stored_function_invalid") unless type == :link || value.has_key?(:bucket) && value.has_key?(:key)
+          @language = "javascript"
+        when String
+          @language = (value =~ /\s*fun\s*\(.*\)\s*->/) ? "erlang" : "javascript"
+        when WalkSpec
+          raise ArgumentError, t("walk_spec_invalid_unless_link") unless type == :link
+        else
+          raise ArgumentError, t("invalid_function_value", :value => value.inspect)
+        end
+        @function = value
+      end
+
+      # Converts the phase to JSON for use while invoking a job.
+      # @return [String] a JSON representation of the phase
+      def to_json(*a)
+        as_json.to_json(*a)
+      end
+
+      # Converts the phase to its JSON-compatible representation for job invocation.
+      # @return [Hash] a Hash-equivalent of the phase
+      def as_json(options = nil)
+        obj = case type
+              when :map, :reduce
+                defaults = {"language" => language, "keep" => keep}
+                case function
+                when Hash
+                  defaults.merge(function)
+                when String
+                  if function =~ /\s*function\s*\(/ || function =~ /\s*fun\s*\(.*\)\s*->/
+                    defaults.merge("source" => function)
+                  else
+                    defaults.merge("name" => function)
+                  end
+                when Array
+                  defaults.merge("module" => function[0], "function" => function[1])
+                end
+              when :link
+                spec = WalkSpec.normalize(function).first
+                {"bucket" => spec.bucket, "tag" => spec.tag, "keep" => spec.keep || keep}
+              end
+        obj["arg"] = arg if arg
+        { type => obj }
+      end
+    end
+  end
+end

data/lib/riak/map_reduce/results.rb

@@ -0,0 +1,49 @@
+module Riak
+  class MapReduce
+    # @api private
+    # Collects and normalizes results from MapReduce requests
+    class Results
+      # Creates a new result collector
+      # @param [MapReduce] mr the MapReduce query
+      def initialize(mr)
+        @keep_count = mr.query.select {|p| p.keep }.size
+        @hash = create_results_hash(mr.query)
+      end
+
+      # Adds a new result to the collector
+      # @param [Fixnum] phase the phase index
+      # @param [Array] result the phase result
+      def add(phase, result)
+        @hash[phase] += result
+      end
+
+      # Coalesces the query results
+      # @return [Array] the query results, coalesced according to the
+      #   phase configuration
+      def report
+        if @keep_count > 1
+          @hash.to_a.sort.map {|(num, results)| results }
+        else
+          @hash[@hash.keys.first]
+        end
+      end
+
+      private
+      def create_results_hash(query)
+        # When the query is empty, only bucket/key pairs are returned,
+        # but implicitly in phase 0.
+        return { 0 => [] } if query.empty?
+
+        # Pre-populate the hash with empty results for kept phases.
+        # Additionally, the last phase is always implictly kept, even
+        # when keep is false.
+        query.inject({}) do |hash, phase|
+          if phase.keep || query[-1] == phase
+            hash[query.index(phase)] = []
+          end
+          hash
+        end
+      end
+    end
+  end
+end

data/lib/riak/map_reduce_error.rb

@@ -0,0 +1,7 @@
+require 'riak/util/translation'
+
+module Riak
+  # Raised when an error occurred in the Javascript map-reduce chain.
+  # The message will be the body of the JSON error response.
+  class MapReduceError < StandardError; end
+end

data/lib/riak/multiget.rb

@@ -0,0 +1,122 @@
+require 'riak/client'
+require 'riak/bucket'
+
+module Riak
+  # Coordinates a parallel fetch operation for multiple values.
+  class Multiget
+    include Util::Translation
+
+    # @return [Riak::Client] the associated client
+    attr_reader :client
+
+    # @return [Array<Bucket, String>] fetch_list an {Array} of {Bucket} and {String} keys to fetch
+    attr_reader :fetch_list
+
+    # @return [Hash<fetch_list_entry, RObject] result_hash a {Hash} of {Bucket} and {String} key pairs to {RObject} instances
+    attr_accessor :result_hash
+
+    # @return [Boolean] finished if the fetch operation has completed
+    attr_reader :finished
+
+    # @return [Integer] The number of threads to use
+    attr_accessor :thread_count
+
+    # Perform a Riak Multiget operation.
+    # @param [Client] client the {Riak::Client} that will perform the multiget
+    # @param [Array<Bucket, String>] fetch_list an {Array} of {Bucket} and {String} keys to fetch
+    # @return [Hash<fetch_list_entry, RObject] result_hash a {Hash} of {Bucket} and {String} key pairs to {RObject} instances
+    def self.get_all(client, fetch_list)
+      multi = new client, fetch_list
+      multi.fetch
+      multi.results
+    end
+
+    # Create a Riak Multiget operation.
+    # @param [Client] client the {Riak::Client} that will perform the multiget
+    # @param [Array<Bucket, String>] fetch_list an {Array} of {Bucket} and {String} keys to fetch
+    def initialize(client, fetch_list)
+      raise ArgumentError, t('client_type', :client => client.inspect) unless client.is_a? Riak::Client
+      raise ArgumentError, t('array_type', :array => fetch_list.inspect) unless fetch_list.is_a? Array
+
+      validate_fetch_list fetch_list
+      @client, @fetch_list = client, fetch_list.uniq
+      self.result_hash = Hash.new
+      @finished = false
+      self.thread_count = client.multiget_threads
+    end
+
+    # Starts the parallelized fetch operation
+    # @raise [ArgumentError] when a non-positive-Integer count is given
+    def fetch
+      queue = fetch_list.dup
+      queue_mutex = Mutex.new
+      result_mutex = Mutex.new
+
+      unless thread_count.is_a?(Integer) && thread_count > 0
+        raise ArgumentError, t("invalid_multiget_thread_count")
+      end
+
+      @threads = 1.upto(thread_count).map do |_node|
+        Thread.new do
+          loop do
+            pair = queue_mutex.synchronize do
+              queue.shift
+            end
+
+            break if pair.nil?
+
+            found = attempt_fetch(*pair)
+            result_mutex.synchronize do
+              result_hash[pair] = found
+            end
+          end
+        end
+      end
+    end
+
+    def results
+      wait_for_finish
+      result_hash
+    end
+
+    def finished?
+      set_finished_for_thread_liveness
+      finished
+    end
+
+    def wait_for_finish
+      return if finished?
+      @threads.each {|t| t.join }
+      @finished = true
+    end
+
+    private
+
+    def attempt_fetch(bucket, key)
+      bucket[key]
+    rescue Riak::FailedRequest => e
+      raise e unless e.not_found?
+      nil
+    end
+
+    def set_finished_for_thread_liveness
+      return if @finished # already done
+
+      all_dead = @threads.none? {|t| t.alive? }
+      return unless all_dead # still working
+
+      @finished = true
+      return
+    end
+
+    def validate_fetch_list(fetch_list)
+      return unless erroneous = fetch_list.detect do |e|
+        bucket, key = e
+        next true unless bucket.is_a? Bucket
+        next true unless key.is_a? String
+      end
+
+      raise ArgumentError, t('fetch_list_type', :problem => erroneous)
+    end
+  end
+end

data/lib/riak/preflist_item.rb

@@ -0,0 +1,7 @@
+require 'riak'
+
+module Riak
+  # @abstract Parent module for preflist items
+  module PreflistItem
+  end
+end

data/lib/riak/rcontent.rb

@@ -0,0 +1,173 @@
+require 'set'
+require 'time'
+require 'yaml'
+require 'forwardable'
+require 'riak/util/translation'
+require 'riak/serializers'
+
+module Riak
+  # Represents single (potentially-conflicted) value stored against a
+  # key in the Riak database. This includes the raw value as well as
+  # metadata.
+  # @since 1.1.0
+  class RContent
+    extend Forwardable
+    include Util::Translation
+
+    # @return [String] the MIME content type of the value
+    attr_accessor :content_type
+
+    # @return [Set<Link>] a Set of {Riak::Link} objects for relationships between this object and other resources
+    attr_accessor :links
+
+    # @return [String] the ETag header from the most recent HTTP response, useful for caching and reloading
+    attr_accessor :etag
+
+    # @return [Time] the Last-Modified header from the most recent HTTP response, useful for caching and reloading
+    attr_accessor :last_modified
+
+    # @return [Hash] a hash of any X-Riak-Meta-* headers that were in the HTTP response, keyed on the trailing portion
+    attr_accessor :meta
+
+    # @return [Hash<Set>] a hash of secondary indexes, where the
+    #   key is the index name and the value is a Set of index
+    #   entries for that index
+    attr_accessor :indexes
+
+    # @return [Riak::RObject] the RObject to which this sibling belongs
+    attr_accessor :robject
+
+    def_delegators :robject, :bucket, :key, :vclock
+
+    # Creates a new object value. This should not normally need to be
+    # called by users of the client. Normal, single-value use can rely
+    # on the delegating accessors on {Riak::RObject}.
+    # @param [RObject] robject the object that this value belongs to
+    # @yield self the new RContent
+    def initialize(robject)
+      @robject = robject
+      @links, @meta = Set.new, {}
+      @indexes = new_index_hash
+      yield self if block_given?
+    end
+
+    def indexes=(hash)
+      @indexes = hash.inject(new_index_hash) do |h, (k, v)|
+        h[k].merge([*v])
+        h
+      end
+    end
+
+    # @return [Object] the unmarshaled form of {#raw_data} stored in riak at this object's key
+    def data
+      if @raw_data && !@data
+        raw = @raw_data.respond_to?(:read) ? @raw_data.read : @raw_data
+        @data = deserialize(raw)
+        @raw_data = nil
+      end
+      @data
+    end
+
+    # @param [Object] new_data unmarshaled form of the data to be stored in
+    #   Riak. Object will be serialized using {#serialize} if a known
+    #   content_type is used. Setting this overrides values stored with
+    #   {#raw_data=}
+    # @return [Object] the object stored
+    def data=(new_data)
+      if new_data.respond_to?(:read)
+        raise ArgumentError.new(t("invalid_io_object"))
+      end
+
+      @raw_data = nil
+      @data = new_data
+    end
+
+
+    # @return [String] raw data stored in riak for this object's key
+    def raw_data
+      if @data && !@raw_data
+        @raw_data = serialize(@data)
+        @data = nil
+      end
+      @raw_data
+    end
+
+    # @param [String, IO-like] new_raw_data the raw data to be stored in Riak
+    #   at this key, will not be marshaled or manipulated prior to storage.
+    #   Overrides any data stored by {#data=}
+    # @return [String] the data stored
+    def raw_data=(new_raw_data)
+      @data = nil
+      @raw_data = new_raw_data
+    end
+
+    # Serializes the internal object data for sending to Riak. Differs based on the content-type.
+    # This method is called internally when storing the object.
+    # Automatically serialized formats:
+    # * JSON (application/json)
+    # * YAML (text/yaml)
+    # * Marshal (application/x-ruby-marshal)
+    # When given an IO-like object (e.g. File), no serialization will
+    # be done.
+    # @param [Object] payload the data to serialize
+    def serialize(payload)
+      Serializers.serialize(@content_type, payload)
+    end
+
+    # Deserializes the internal object data from a Riak response. Differs based on the content-type.
+    # This method is called internally when loading the object.
+    # Automatically deserialized formats:
+    # * JSON (application/json)
+    # * YAML (text/yaml)
+    # * Marshal (application/x-ruby-marshal)
+    # @param [String] body the serialized response body
+    def deserialize(body)
+      Serializers.deserialize(@content_type, body)
+    end
+
+    # @return [String] A representation suitable for IRB and debugging output
+    def inspect
+      body = if @data || Serializers[content_type]
+               data.inspect
+             else
+               @raw_data && "(#{@raw_data.size} bytes)"
+             end
+      "#<#{self.class.name} [#{@content_type}]:#{body}>"
+    end
+
+    # @api private
+    def load_map_reduce_value(hash)
+      metadata = hash['metadata']
+      extract_if_present(metadata, 'X-Riak-VTag', :etag)
+      extract_if_present(metadata, 'content-type', :content_type)
+      extract_if_present(metadata, 'X-Riak-Last-Modified', :last_modified) { |v| Time.httpdate( v ) }
+      extract_if_present(metadata, 'index', :indexes) do |entries|
+        Hash[ entries.map {|k, v| [k, Set.new(Array(v))] } ]
+      end
+      extract_if_present(metadata, 'Links', :links) do |links|
+        Set.new( links.map { |l| Link.new(*l) } )
+      end
+      extract_if_present(metadata, 'X-Riak-Meta', :meta) do |meta|
+        Hash[
+             meta.map do |k, v|
+               [k.sub(%r{^x-riak-meta-}i, ''), [v]]
+             end
+            ]
+      end
+      extract_if_present(hash, 'data', :data) { |v| deserialize(v) }
+    end
+
+    private
+    def extract_if_present(hash, key, attribute = nil)
+      if hash[key].present?
+        attribute ||= key
+        value = block_given? ? yield(hash[key]) : hash[key]
+        send("#{attribute}=", value)
+      end
+    end
+
+    def new_index_hash
+      Hash.new {|h, k| h[k] = Set.new }
+    end
+  end
+end