seomoz-riak-client 1.0.0.pre

data/lib/riak/encoding.rb

@@ -0,0 +1,6 @@
+if defined? Encoding
+  Encoding.default_internal = "UTF-8" if Encoding.default_internal.nil? ||
+                                        !Encoding.default_internal.ascii_compatible?
+else
+  $KCODE = "U"
+end

data/lib/riak/failed_request.rb

@@ -0,0 +1,81 @@
+
+require 'riak/util/translation'
+require 'riak/json'
+
+module Riak
+  # Exception raised when receiving an unexpected client response from
+  # Riak.
+  class FailedRequest < StandardError
+    include Util::Translation
+
+    def initialize(message)
+      super(message ||  t('failed_request'))
+    end
+  end
+
+  # Exception raised when the expected HTTP response code from Riak
+  # fails to match the actual response code.
+  class HTTPFailedRequest < FailedRequest
+    # @return [Symbol] the HTTP method, one of :head, :get, :post, :put, :delete
+    attr_reader :method
+    # @return [Fixnum] the expected response code
+    attr_reader :expected
+    # @return [Fixnum] the received response code
+    attr_reader :code
+    # @return [Hash] the response headers
+    attr_reader :headers
+    # @return [String] the response body, if present
+    attr_reader :body
+
+    def initialize(method, expected_code, received_code, headers, body)
+      @method, @expected, @code, @headers, @body = method, expected_code, received_code, headers, body
+      super t("http_failed_request", :expected => @expected.inspect, :code => @code, :body => @body)
+    end
+
+    def is_json?
+      headers['content-type'].include?('application/json')
+    end
+    
+    # @return [true,false] whether the error represents a "not found" response
+    def not_found?
+      @code.to_i == 404
+    end
+
+    # @return [true,false] whether the error represents an internal
+    #   server error
+    def server_error?
+      @code.to_i == 500
+    end
+  end
+
+  # Exception raised when receiving an unexpected Protocol Buffers response from Riak
+  class ProtobuffsFailedRequest < FailedRequest
+    def initialize(code, message)
+      super t('protobuffs_failed_request', :code => code, :body => message)
+      @original_message = message
+      @not_found = code == :not_found
+      @server_error = code == :server_error
+    end
+
+    # @return [true, false] whether the error response is in JSON
+    def is_json?
+      begin
+        JSON.parse(original_message)
+        true
+      rescue
+        false
+      end
+    end
+    
+    # @return [true,false] whether the error represents a "not found" response
+    def not_found?
+      @not_found
+    end
+
+    # @return [true,false] whether the error represents an internal
+    #   server error
+    def server_error?
+      @server_error
+    end
+  end
+end

data/lib/riak/i18n.rb

@@ -0,0 +1,3 @@
+require 'i18n'
+
+I18n.load_path << File.expand_path("../locale/en.yml", __FILE__)

data/lib/riak/json.rb

@@ -0,0 +1,28 @@
+require 'multi_json'
+MultiJson.engine # Force loading of an engine
+require 'riak/core_ext/json'
+
+module Riak
+  class << self
+    # Options that will be passed to the JSON parser and encoder.
+    # Defaults to {:max_nesting => 20}
+    attr_accessor :json_options
+  end
+  self.json_options = {:max_nesting => 20}
+
+  # JSON module for internal use inside riak-client
+  module JSON
+    class << self
+      # Parse a JSON string
+      def parse(str)
+        MultiJson.decode(str, Riak.json_options)
+      end
+
+      # Generate a JSON string
+      def encode(obj)
+        MultiJson.encode(obj)
+      end
+      alias :dump :encode
+    end
+  end
+end

data/lib/riak/link.rb

@@ -0,0 +1,85 @@
+
+require 'riak/util/translation'
+require 'riak/util/escape'
+require 'riak/walk_spec'
+
+module Riak
+  # Represents a link from one object to another in Riak
+  class Link
+    include Util::Translation
+    include Util::Escape
+
+    # @return [String] the relationship tag (or "rel") of the other resource to this one
+    attr_accessor :tag
+    alias_method :rel, :tag
+    alias_method :rel=, :tag=
+
+    # @return [String] the bucket of the related resource
+    attr_accessor :bucket
+
+    # @return [String] the key of the related resource
+    attr_accessor :key
+
+    %w{bucket key}.each do |m|
+      class_eval %{ def #{m}=(value); @url = nil; @#{m} = value; end }
+    end
+
+    # @param [String] header_string the string value of the Link: HTTP header from a Riak response
+    # @return [Array<Link>] an array of Riak::Link structs parsed from the header
+    def self.parse(header_string)
+      header_string.scan(%r{<([^>]+)>\s*;\s*(?:rel|riaktag)=\"([^\"]+)\"}).map do |match|
+        new(match[0], match[1])
+      end
+    end
+
+    # @overload initialize(url, tag)
+    #   @param [String] url the url of the related resource
+    #   @param [String] tag the tag for the related resource
+    # @overload initialize(bucket, key, tag)
+    #   @param [String] bucket the bucket of the related resource
+    #   @param [String] key the key of the related resource
+    #   @param [String] tag the tag for the related resource
+    def initialize(*args)
+      raise ArgumentError unless (2..3).include?(args.size)
+      if args.size == 2
+        self.url, @tag = args
+      else
+        @bucket, @key, @tag = args
+      end
+    end
+
+    # @return [String] the URL (relative or absolute) of the related resource
+    def url
+      @url ||= "/riak/#{escape(bucket)}" + (key.blank? ? "" : "/#{escape(key)}")
+    end
+
+    def url=(value)
+      @url = value
+      @bucket = unescape($1) if value =~ %r{^/[^/]+/([^/]+)/?}
+      @key = unescape($1) if value =~ %r{^/[^/]+/[^/]+/([^/]+)/?}
+    end
+
+    def inspect; to_s; end
+
+    def to_s
+      %Q[<#{url}>; riaktag="#{tag}"]
+    end
+
+    def hash
+      self.to_s.hash
+    end
+
+    def eql?(other)
+      self == other
+    end
+
+    def ==(other)
+      other.is_a?(Link) && url == other.url && tag == other.tag
+    end
+
+    def to_walk_spec
+      raise t("bucket_link_conversion") if tag == "up" || key.nil?
+      WalkSpec.new(:bucket => bucket, :tag => tag)
+    end
+  end
+end

data/lib/riak/locale/en.yml

@@ -0,0 +1,48 @@
+en:
+  riak:
+    backwards_clock: "System clock moved backwards, ID generation will fail for %{delay} more milliseconds."
+    bucket_link_conversion: "Can't convert a bucket link to a walk spec"
+    client_type: "invalid argument %{client} is not a Riak::Client"
+    conflict_resolver_invalid: "The given resolver (%{resolver}) did not respond to :call"
+    content_type_undefined: "content_type is not defined!"
+    deprecated:
+      port: "DEPRECATION: Riak::Client#port has been deprecated, use #http_port or #pb_port for the appropriate protocol.\n%{backtrace}"
+      search: "DEPRECATION: Riak Search features are included in the main client, you no longer need to require 'riak/search'.\n%{backtrace}"
+    empty_map_reduce_query: "Specify one or more query phases to your MapReduce."
+    failed_request: "Client request failed."
+    filter_needs_block: "Filter %{filter} expects a block."
+    filter_arity_mismatch: "Filter %{filter} expects %{expected} arguments but %{received} were given."
+    full_bucket_mapred: "Full-bucket MapReduce, including key filters, invokes list-keys which is an expensive operation that should not be used in production.\n    %{backtrace}"
+    hash_type: "invalid argument %{hash} is not a Hash"
+    http_configuration: "The %{backend} HTTP backend cannot be used. Please check its requirements."
+    http_failed_request: "Expected %{expected} from Riak but received %{code}. %{body}"
+    hostname_invalid: "host must be a valid hostname"
+    protocol_invalid: "'%{invalid}' is not a valid protocol, valid values are %{valid}"
+    invalid_basic_auth: "basic auth must be set using 'user:pass'"
+    invalid_client_id: "Invalid client ID, must be a string or between 0 and %{max_id}"
+    invalid_io_object: "Invalid IO-like object assigned to RObject#data.  It should be assigned to raw_data instead."
+    invalid_function_value: "invalid value for function: %{value}"
+    invalid_options: "Invalid configuration options given."
+    invalid_phase_type: "type must be :map, :reduce, or :link"
+    invalid_ssl_verify_mode: "%{invalid} is not a valid :verify_mode option for SSL. Valid options are 'peer' and 'none'."
+    loading_bucket: "while loading bucket '%{name}'"    
+    list_buckets: "Riak::Client#buckets is an expensive operation that should not be used in production.\n    %{backtrace}"
+    list_keys: "Riak::Bucket#keys is an expensive operation that should not be used in production.\n    %{backtrace}"
+    missing_block: "A block must be given."
+    missing_host_and_port: "You must specify a host and port, or use the defaults of 127.0.0.1:8098"
+    module_function_pair_required: "function must have two elements when an array"
+    not_found: "The requested object was not found."
+    path_and_body_required: "You must supply both a resource path and a body."
+    port_invalid: "port must be an integer between 0 and 65535"
+    protobuffs_failed_request: "Expected success from Riak but received %{code}. %{body}"
+    request_body_type: "Request body must be a String or respond to :read."
+    resource_path_short: "Resource path too short"
+    search_docs_require_id: "Search index documents must include the 'id' field."
+    search_remove_requires_id_or_query: "Search index documents to be removed must have 'id' or 'query' keys."
+    serializer_not_implemented: "No serializer has been registered for content type %{content_type}"
+    stale_write_prevented: "Stale write prevented by client."
+    stored_function_invalid: "function must have :bucket and :key when a hash"
+    string_type: "invalid_argument %{string} is not a String"
+    too_few_arguments: "too few arguments: %{params}"
+    walk_spec_invalid_unless_link: "WalkSpec is only valid for a function when the type is :link"
+    wrong_argument_count_walk_spec: "wrong number of arguments (one Hash or bucket,tag,keep required)"

data/lib/riak/map_reduce.rb

@@ -0,0 +1,206 @@
+
+require 'riak/util/translation'
+require 'riak/util/escape'
+require 'riak/json'
+require 'riak/client'
+require 'riak/bucket'
+require 'riak/robject'
+require 'riak/walk_spec'
+require 'riak/failed_request'
+require 'riak/map_reduce_error'
+require 'riak/map_reduce/phase'
+require 'riak/map_reduce/filter_builder'
+
+module Riak
+  # Class for invoking map-reduce jobs using the HTTP interface.
+  class MapReduce
+    include Util::Translation
+    include Util::Escape
+
+    # @return [Array<[bucket,key]>,String,Hash<:bucket,:filters>] The
+    #       bucket/keys for input to the job, or the bucket (all
+    #       keys), or a hash containing the bucket and key-filters.
+    # @see #add
+    attr_accessor :inputs
+
+    # @return [Array<Phase>] The map and reduce phases that will be executed
+    # @see #map
+    # @see #reduce
+    # @see #link
+    attr_accessor :query
+
+    # Creates a new map-reduce job.
+    # @param [Client] client the Riak::Client interface
+    # @yield [self] helpful for initializing the job
+    def initialize(client)
+      @client, @inputs, @query = client, [], []
+      yield self if block_given?
+    end
+
+    # Add or replace inputs for the job.
+    # @overload add(bucket)
+    #   Run the job across all keys in the bucket.  This will replace any other inputs previously added.
+    #   @param [String, Bucket] bucket the bucket to run the job on
+    # @overload add(bucket,key)
+    #   Add a bucket/key pair to the job.
+    #   @param [String,Bucket] bucket the bucket of the object
+    #   @param [String] key the key of the object
+    # @overload add(object)
+    #   Add an object to the job (by its bucket/key)
+    #   @param [RObject] object the object to add to the inputs
+    # @overload add(bucket, key, keydata)
+    #   @param [String,Bucket] bucket the bucket of the object
+    #   @param [String] key the key of the object
+    #   @param [String] keydata extra data to pass along with the object to the job
+    # @overload add(bucket, filters)
+    #   Run the job across all keys in the bucket, with the given
+    #   key-filters. This will replace any other inputs previously
+    #   added. (Requires Riak 0.14)
+    #   @param [String,Bucket] bucket the bucket to filter keys from
+    #   @param [Array<Array>] filters a list of key-filters to apply
+    #                                 to the key list
+    # @return [MapReduce] self
+    def add(*params)
+      params = params.dup
+      params = params.first if Array === params.first
+      case params.size
+      when 1
+        p = params.first
+        case p
+        when Bucket
+          warn(t('full_bucket_mapred', :backtrace => caller.join("\n    "))) unless Riak.disable_list_keys_warnings
+          @inputs = escape(p.name)
+        when RObject
+          @inputs << [escape(p.bucket.name), escape(p.key)]
+        when String
+          warn(t('full_bucket_mapred', :backtrace => caller.join("\n    "))) unless Riak.disable_list_keys_warnings
+          @inputs = escape(p)
+        end
+      when 2..3
+        bucket = params.shift
+        bucket = bucket.name if Bucket === bucket
+        if Array === params.first
+          warn(t('full_bucket_mapred', :backtrace => caller.join("\n    "))) unless Riak.disable_list_keys_warnings
+          @inputs = {:bucket => escape(bucket), :key_filters => params.first }
+        else
+          key = params.shift
+          @inputs << params.unshift(escape(key)).unshift(escape(bucket))
+        end
+      end
+      self
+    end
+    alias :<< :add
+    alias :include :add
+
+    # Adds a bucket and key-filters built by the given
+    # block. Equivalent to #add with a list of filters.
+    # @param [String] bucket the bucket to apply key-filters to
+    # @yield [] builder block - instance_eval'ed into a FilterBuilder
+    # @return [MapReduce] self
+    # @see MapReduce#add
+    def filter(bucket, &block)
+      add(bucket, FilterBuilder.new(&block).to_a)
+    end
+
+    # (Riak Search) Use a search query to start a map/reduce job.
+    # @param [String, Bucket] bucket the bucket/index to search
+    # @param [String] query the query to run
+    # @return [MapReduce] self
+    def search(bucket, query)
+      bucket = bucket.name if bucket.respond_to?(:name)
+      @inputs = {:module => "riak_search", :function => "mapred_search", :arg => [bucket, query]}
+      self
+    end
+
+    # Add a map phase to the job.
+    # @overload map(function)
+    #   @param [String, Array] function a Javascript function that represents the phase, or an Erlang [module,function] pair
+    # @overload map(function?, options)
+    #   @param [String, Array] function a Javascript function that represents the phase, or an Erlang [module, function] pair
+    #   @param [Hash] options extra options for the phase (see {Phase#initialize})
+    # @return [MapReduce] self
+    # @see Phase#initialize
+    def map(*params)
+      options = params.extract_options!
+      @query << Phase.new({:type => :map, :function => params.shift}.merge(options))
+      self
+    end
+
+    # Add a reduce phase to the job.
+    # @overload reduce(function)
+    #   @param [String, Array] function a Javascript function that represents the phase, or an Erlang [module,function] pair
+    # @overload reduce(function?, options)
+    #   @param [String, Array] function a Javascript function that represents the phase, or an Erlang [module, function] pair
+    #   @param [Hash] options extra options for the phase (see {Phase#initialize})
+    # @return [MapReduce] self
+    # @see Phase#initialize
+    def reduce(*params)
+      options = params.extract_options!
+      @query << Phase.new({:type => :reduce, :function => params.shift}.merge(options))
+      self
+    end
+
+    # Add a link phase to the job. Link phases follow links attached to objects automatically (a special case of map).
+    # @overload link(walk_spec, options={})
+    #   @param [WalkSpec] walk_spec a WalkSpec that represents the types of links to follow
+    #   @param [Hash] options extra options for the phase (see {Phase#initialize})
+    # @overload link(bucket, tag, keep, options={})
+    #   @param [String, nil] bucket the bucket to limit links to
+    #   @param [String, nil] tag the tag to limit links to
+    #   @param [Boolean] keep whether to keep results of this phase (overrides the phase options)
+    #   @param [Hash] options extra options for the phase (see {Phase#initialize})
+    # @overload link(options)
+    #   @param [Hash] options options for both the walk spec and link phase
+    #   @see WalkSpec#initialize
+    # @return [MapReduce] self
+    # @see Phase#initialize
+    def link(*params)
+      options = params.extract_options!
+      walk_spec_options = options.slice!(:type, :function, :language, :arg) unless params.first
+      walk_spec = WalkSpec.normalize(params.shift || walk_spec_options).first
+      @query << Phase.new({:type => :link, :function => walk_spec}.merge(options))
+      self
+    end
+
+    # Sets the timeout for the map-reduce job.
+    # @param [Fixnum] value the job timeout, in milliseconds
+    def timeout(value)
+      @timeout = value
+      return self
+    end
+    alias :timeout= :timeout
+
+    # Convert the job to JSON for submission over the HTTP interface.
+    # @return [String] the JSON representation
+    def to_json(*a)
+      hash = {"inputs" => inputs, "query" => query.map(&:as_json)}
+      hash['timeout'] = @timeout.to_i if @timeout
+      hash.to_json(*a)
+    end
+
+    # Executes this map-reduce job.
+    # @overload run
+    #   Return the entire collection of results.
+    #   @return [Array<Array>] similar to link-walking, each element is
+    #     an array of results from a phase where "keep" is true. If there
+    #     is only one "keep" phase, only the results from that phase will
+    #     be returned.
+    # @overload run
+    #   Stream the results through the given block without accumulating.
+    #   @yield [phase, data] A block to stream results through
+    #   @yieldparam [Fixnum] phase the phase from which the results were
+    #          generated
+    #   @yieldparam [Array] data a list of results from the phase
+    #   @return [nil] nothing
+    def run(&block)
+      raise MapReduceError.new(t("empty_map_reduce_query")) if @query.empty?
+      @client.backend.mapred(self, &block)
+    rescue FailedRequest => fr
+      if fr.server_error? && fr.is_json?
+        raise MapReduceError.new(fr.body)
+      else
+        raise fr
+      end
+    end
+  end
+end