ripple 0.5.0

data/lib/riak/locale/en.yml

@@ -0,0 +1,37 @@
+# Copyright 2010 Sean Cribbs, Sonian Inc., and Basho Technologies, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+en:
+  riak:
+    client_type: "invalid argument {{client}} is not a Riak::Client"
+    string_type: "invalid_argument {{string}} is not a String"
+    loading_bucket: "while loading bucket '{{name}}'"
+    failed_request: "Expected {{expected}} from Riak but received {{code}}. {{body}}"
+    hash_type: "invalid argument {{hash}} is not a Hash"
+    path_and_body_required: "You must supply both a resource path and a body."
+    request_body_type: "Request body must be a string or IO."
+    resource_path_short: "Resource path too short"
+    missing_host_and_port: "You must specify a host and port, or use the defaults of 127.0.0.1:8098"
+    invalid_client_id: "Invalid client ID, must be a string or between 0 and {{max_id}}"
+    hostname_invalid: "host must be a valid hostname"
+    port_invalid: "port must be an integer between 0 and 65535"
+    install_curb: "curb library not found! Please `gem install curb` for better performance."
+    bucket_link_conversion: "Can't convert a bucket link to a walk spec"
+    invalid_phase_type: "type must be :map, :reduce, or :link"
+    module_function_pair_required: "function must have two elements when an array"
+    stored_function_invalid: "function must have :bucket and :key when a hash"
+    walk_spec_invalid_unless_link: "WalkSpec is only valid for a function when the type is :link"
+    invalid_function_value: "invalid value for function: {{value}}"
+    content_type_undefined: "content_type is not defined!"
+    too_few_arguments: "too few arguments: {{params}}"
+    wrong_argument_count_walk_spec: "wrong number of arguments (one Hash or bucket,tag,keep required)"

data/lib/riak/map_reduce.rb

@@ -0,0 +1,240 @@
+# Copyright 2010 Sean Cribbs, Sonian Inc., and Basho Technologies, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+require 'riak'
+
+module Riak
+  # Class for invoking map-reduce jobs using the HTTP interface.
+  class MapReduce
+    include Util::Translation
+    # @return [Array<[bucket,key]>,String] The bucket/keys for input to the job, or the bucket (all keys).
+    # @see #add
+    attr_accessor :inputs
+
+    # @return [Array<Phase>] The map and reduce phases that will be executed
+    # @see #map
+    # @see #reduce
+    # @see #link
+    attr_accessor :query
+
+    # Creates a new map-reduce job.
+    # @param [Client] client the Riak::Client interface
+    # @yield [self] helpful for initializing the job
+    def initialize(client)
+      @client, @inputs, @query = client, [], []
+      yield self if block_given?
+    end
+
+    # Add or replace inputs for the job.
+    # @overload add(bucket)
+    #   Run the job across all keys in the bucket.  This will replace any other inputs previously added.
+    #   @param [String, Bucket] bucket the bucket to run the job on
+    # @overload add(bucket,key)
+    #   Add a bucket/key pair to the job.
+    #   @param [String,Bucket] bucket the bucket of the object
+    #   @param [String] key the key of the object
+    # @overload add(object)
+    #   Add an object to the job (by its bucket/key)
+    #   @param [RObject] object the object to add to the inputs
+    # @overload add(bucket, key, keydata)
+    #   @param [String,Bucket] bucket the bucket of the object
+    #   @param [String] key the key of the object
+    #   @param [String] keydata extra data to pass along with the object to the job
+    # @return [MapReduce] self
+    def add(*params)
+      params = params.dup.flatten
+      case params.size
+      when 1
+        p = params.first
+        case p
+        when Bucket
+          @inputs = p.name
+        when RObject
+          @inputs << [p.bucket.name, p.key]
+        when String
+          @inputs = p
+        end
+      when 2..3
+        bucket = params.shift
+        bucket = bucket.name if Bucket === bucket
+        @inputs << params.unshift(bucket)
+      end
+      self
+    end
+    alias :<< :add
+    alias :include :add
+
+    # Add a map phase to the job.
+    # @overload map(function)
+    #   @param [String, Array] function a Javascript function that represents the phase, or an Erlang [module,function] pair
+    # @overload map(function?, options)
+    #   @param [String, Array] function a Javascript function that represents the phase, or an Erlang [module, function] pair
+    #   @param [Hash] options extra options for the phase (see {Phase#initialize})
+    # @return [MapReduce] self
+    # @see Phase#initialize
+    def map(*params)
+      options = params.extract_options!
+      @query << Phase.new({:type => :map, :function => params.shift}.merge(options))
+      self
+    end
+
+    # Add a reduce phase to the job.
+    # @overload reduce(function)
+    #   @param [String, Array] function a Javascript function that represents the phase, or an Erlang [module,function] pair
+    # @overload reduce(function?, options)
+    #   @param [String, Array] function a Javascript function that represents the phase, or an Erlang [module, function] pair
+    #   @param [Hash] options extra options for the phase (see {Phase#initialize})
+    # @return [MapReduce] self
+    # @see Phase#initialize
+    def reduce(*params)
+      options = params.extract_options!
+      @query << Phase.new({:type => :reduce, :function => params.shift}.merge(options))
+      self
+    end
+
+    # Add a link phase to the job. Link phases follow links attached to objects automatically (a special case of map).
+    # @overload link(walk_spec, options={})
+    #   @param [WalkSpec] walk_spec a WalkSpec that represents the types of links to follow
+    #   @param [Hash] options extra options for the phase (see {Phase#initialize})
+    # @overload link(bucket, tag, keep, options={})
+    #   @param [String, nil] bucket the bucket to limit links to
+    #   @param [String, nil] tag the tag to limit links to
+    #   @param [Boolean] keep whether to keep results of this phase (overrides the phase options)
+    #   @param [Hash] options extra options for the phase (see {Phase#initialize})
+    # @overload link(options)
+    #   @param [Hash] options options for both the walk spec and link phase
+    #   @see WalkSpec#initialize
+    # @return [MapReduce] self
+    # @see Phase#initialize
+    def link(*params)
+      options = params.extract_options!
+      walk_spec_options = options.slice!(:type, :function, :language, :arg) unless params.first
+      walk_spec = WalkSpec.normalize(params.shift || walk_spec_options).first
+      @query << Phase.new({:type => :link, :function => walk_spec}.merge(options))
+      self
+    end
+
+    # Convert the job to JSON for submission over the HTTP interface.
+    # @return [String] the JSON representation
+    def to_json(options={})
+      ActiveSupport::JSON.encode({"inputs" => inputs, "query" => query.map(&:as_json)}, options)
+    end
+
+    # Executes this map-reduce job.
+    # @return [Array<Array>] similar to link-walking, each element is an array of results from a phase where "keep" is true. If there is only one "keep" phase, only the results from that phase will be returned.
+    def run
+      response = @client.http.post(200, @client.mapred, to_json, {"Content-Type" => "application/json", "Accept" => "application/json"})
+      if response.try(:[], :headers).try(:[],'content-type').include?("application/json")
+        JSON.parse(response[:body])
+      else
+        response
+      end
+    rescue FailedRequest => fr
+      if fr.code == 500 && fr.headers['content-type'].include?("application/json")
+        raise MapReduceError.new(fr.body)
+      else
+        raise fr
+      end
+    end
+
+    # Represents an individual phase in a map-reduce pipeline. Generally you'll want to call
+    # methods of {MapReduce} instead of using this directly.
+    class Phase
+      include Util::Translation
+      # @return [Symbol] the type of phase - :map, :reduce, or :link
+      attr_accessor :type
+
+      # @return [String, Array<String, String>, Hash, WalkSpec] For :map and :reduce types, the Javascript function to run (as a string or hash with bucket/key), or the module + function in Erlang to run. For a :link type, a {Riak::WalkSpec} or an equivalent hash.
+      attr_accessor :function
+
+      # @return [String] the language of the phase's function - "javascript" or "erlang". Meaningless for :link type phases.
+      attr_accessor :language
+
+      # @return [Boolean] whether results of this phase will be returned
+      attr_accessor :keep
+
+      # @return [Array] any extra static arguments to pass to the phase
+      attr_accessor :arg
+
+      # Creates a phase in the map-reduce pipeline
+      # @param [Hash] options options for the phase
+      # @option options [Symbol] :type one of :map, :reduce, :link
+      # @option options [String] :language ("javascript") "erlang" or "javascript"
+      # @option options [String, Array, Hash] :function In the case of Javascript, a literal function in a string, or a hash with :bucket and :key. In the case of Erlang, an Array of [module, function].  For a :link phase, a hash including any of :bucket, :tag or a WalkSpec.
+      # @option options [Boolean] :keep (false) whether to return the results of this phase
+      # @option options [Array] :arg (nil) any extra static arguments to pass to the phase
+      def initialize(options={})
+        self.type = options[:type]
+        self.language = options[:language] || "javascript"
+        self.function = options[:function]
+        self.keep = options[:keep] || false
+        self.arg = options[:arg]
+      end
+
+      def type=(value)
+        raise ArgumentError, t("invalid_phase_type") unless value.to_s =~ /^(map|reduce|link)$/i
+        @type = value.to_s.downcase.to_sym
+      end
+
+      def function=(value)
+        case value
+        when Array
+          raise ArgumentError, t("module_function_pair_required") unless value.size == 2
+          @language = "erlang"
+        when Hash
+          raise ArgumentError, t("stored_function_invalid") unless type == :link || value.has_key?(:bucket) && value.has_key?(:key)
+          @language = "javascript"
+        when String
+          @language = "javascript"
+        when WalkSpec
+          raise ArgumentError, t("walk_spec_invalid_unless_link") unless type == :link
+        else
+          raise ArgumentError, t("invalid_function_value", :value => value.inspect)
+        end
+        @function = value
+      end
+
+      # Converts the phase to JSON for use while invoking a job.
+      # @return [String] a JSON representation of the phase
+      def to_json(options=nil)
+        ActiveSupport::JSON.encode(as_json, options)
+      end
+
+      # Converts the phase to its JSON-compatible representation for job invocation.
+      # @return [Hash] a Hash-equivalent of the phase
+      def as_json(options=nil)
+        obj = case type
+              when :map, :reduce
+                defaults = {"language" => language, "keep" => keep}
+                case function
+                when Hash
+                  defaults.merge(function)
+                when String
+                  if function =~ /\s*function/
+                    defaults.merge("source" => function)
+                  else
+                    defaults.merge("name" => function)
+                  end
+                when Array
+                  defaults.merge("module" => function[0], "function" => function[1])
+                end
+              when :link
+                spec = WalkSpec.normalize(function).first
+                {"bucket" => spec.bucket, "tag" => spec.tag, "keep" => spec.keep || keep}
+              end
+        obj["arg"] = arg if arg
+        { type => obj }
+      end
+    end
+  end
+end

data/lib/riak/map_reduce_error.rb

@@ -0,0 +1,20 @@
+# Copyright 2010 Sean Cribbs, Sonian Inc., and Basho Technologies, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+require 'riak'
+
+module Riak
+  # Raised when an error occurred in the Javascript map-reduce chain.
+  # The message will be the body of the JSON error response.
+  class MapReduceError < StandardError; end
+end

data/lib/riak/robject.rb

@@ -0,0 +1,234 @@
+# Copyright 2010 Sean Cribbs, Sonian Inc., and Basho Technologies, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+require 'riak'
+
+module Riak
+  # Parent class of all object types supported by ripple. {Riak::RObject} represents
+  # the data and metadata stored in a bucket/key pair in the Riak database.
+  class RObject
+    include Util
+    include Util::Translation
+
+    # @return [Bucket] the bucket in which this object is contained
+    attr_accessor :bucket
+
+    # @return [String] the key of this object within its bucket
+    attr_accessor :key
+
+    # @return [String] the MIME content type of the object
+    attr_accessor :content_type
+
+    # @return [String] the Riak vector clock for the object
+    attr_accessor :vclock
+    alias_attribute :vector_clock, :vclock
+
+    # @return [Object] the data stored in Riak at this object's key. Varies in format by content-type, defaulting to String from the response body.
+    attr_accessor :data
+
+    # @return [Array<Link>] an array of {Riak::Link} objects for relationships between this object and other resources
+    attr_accessor :links
+
+    # @return [String] the ETag header from the most recent HTTP response, useful for caching and reloading
+    attr_accessor :etag
+
+    # @return [Time] the Last-Modified header from the most recent HTTP response, useful for caching and reloading
+    attr_accessor :last_modified
+
+    # @return [Hash] a hash of any X-Riak-Meta-* headers that were in the HTTP response, keyed on the trailing portion
+    attr_accessor :meta
+
+    # Create a new object manually
+    # @param [Bucket] bucket the bucket in which the object exists
+    # @param [String] key the key at which the object resides. If nil, a key will be assigned when the object is saved.
+    # @see Bucket#get
+    def initialize(bucket, key=nil)
+      @bucket, @key = bucket, key
+      @links, @meta = [], {}
+    end
+
+    # Load object data from an HTTP response
+    # @param [Hash] response a response from {Riak::Client::HTTPBackend}
+    def load(response)
+      extract_header(response, "location", :key) {|v| URI.unescape(v.split("/").last) }
+      extract_header(response, "content-type", :content_type)
+      extract_header(response, "x-riak-vclock", :vclock)
+      extract_header(response, "link", :links) {|v| Link.parse(v) }
+      extract_header(response, "etag", :etag)
+      extract_header(response, "last-modified", :last_modified) {|v| Time.httpdate(v) }
+      @meta = response[:headers].inject({}) do |h,(k,v)|
+        if k =~ /x-riak-meta-(.*)/
+          h[$1] = v
+        end
+        h
+      end
+      @data = deserialize(response[:body]) if response[:body].present?
+      self
+    end
+
+    # HTTP header hash that will be sent along when storing the object
+    # @return [Hash] hash of HTTP Headers
+    def store_headers
+      {}.tap do |hash|
+        hash["Content-Type"] = @content_type
+        hash["X-Riak-Vclock"] = @vclock if @vclock
+        unless @links.blank?
+          hash["Link"] = @links.reject {|l| l.rel == "up" }.map(&:to_s).join(", ")
+        end
+        unless @meta.blank?
+          @meta.each do |k,v|
+            hash["X-Riak-Meta-#{k}"] = v.to_s
+          end
+        end
+      end
+    end
+
+    # HTTP header hash that will be sent along when reloading the object
+    # @return [Hash] hash of HTTP headers
+    def reload_headers
+      {}.tap do |h|
+        h['If-None-Match'] = @etag if @etag.present?
+        h['If-Modified-Since'] = @last_modified.httpdate if @last_modified.present?
+      end
+    end
+
+    # Store the object in Riak
+    # @param [Hash] options query parameters
+    # @option options [Fixnum] :r the "r" parameter (Read quorum for the implicit read performed when validating the store operation)
+    # @option options [Fixnum] :w the "w" parameter (Write quorum)
+    # @option options [Fixnum] :dw the "dw" parameter (Durable-write quorum)
+    # @option options [Boolean] :returnbody (true) whether to return the result of a successful write in the body of the response. Set to false for fire-and-forget updates, set to true to immediately have access to the object's stored representation.
+    # @return [Riak::RObject] self
+    # @raise [ArgumentError] if the content_type is not defined
+    def store(options={})
+      raise ArgumentError, t("content_type_undefined") unless @content_type.present?
+      params = {:returnbody => true}.merge(options)
+      method, codes, path = @key.present? ? [:put, [200,204], "#{@bucket.name}/#{@key}"] : [:post, 201, @bucket.name]
+      response = @bucket.client.http.send(method, codes, @bucket.client.prefix, path, params, serialize(data), store_headers)
+      load(response)
+    end
+
+    # Reload the object from Riak.  Will use conditional GETs when possible.
+    # @param [Hash] options query parameters
+    # @option options [Fixnum] :r the "r" parameter (Read quorum)
+    # @option options [Boolean] :force will force a reload request if the vclock is not present, useful for reloading the object after a store (not passed in the query params)
+    # @return [Riak::RObject] self
+    def reload(options={})
+      force = options.delete(:force)
+      return self unless @key && (@vclock || force)
+      response = @bucket.client.http.get([200, 304], @bucket.client.prefix, @bucket.name, @key, options, reload_headers)
+      load(response) if response[:code] == 200
+      self
+    end
+
+    alias :fetch :reload
+
+    # Delete the object from Riak and freeze this instance.  Will work whether or not the object actually
+    # exists in the Riak database.
+    def delete
+      return if key.blank?
+      @bucket.client.http.delete([204,404], @bucket.client.prefix, @bucket.name, key)
+      freeze
+    end
+
+    # Serializes the internal object data for sending to Riak. Differs based on the content-type.
+    # This method is called internally when storing the object.
+    # Automatically serialized formats:
+    # * JSON (application/json)
+    # * YAML (text/yaml)
+    # * Marshal (application/octet-stream if meta['ruby-serialization'] == "Marshal")
+    # @param [Object] payload the data to serialize
+    def serialize(payload)
+      return payload if IO === payload
+      case @content_type
+      when /json/
+        ActiveSupport::JSON.encode(payload)
+      when /yaml/
+        YAML.dump(payload)
+      when "application/octet-stream"
+        if @meta['ruby-serialization'] == "Marshal"
+          Marshal.dump(payload)
+        else
+          payload.to_s
+        end
+      else
+        payload.to_s
+      end
+    end
+
+    # Deserializes the internal object data from a Riak response. Differs based on the content-type.
+    # This method is called internally when loading the object.
+    # Automatically deserialized formats:
+    # * JSON (application/json)
+    # * YAML (text/yaml)
+    # * Marshal (application/octet-stream if meta['ruby-serialization'] == "Marshal")
+    # @param [String] body the serialized response body
+    def deserialize(body)
+      case @content_type
+      when /json/
+        ActiveSupport::JSON.decode(body)
+      when /yaml/
+        YAML.load(body)
+      when "application/octet-stream"
+        if @meta['ruby-serialization'] == "Marshal"
+          Marshal.load(body)
+        else
+          body
+        end
+      else
+        body
+      end
+    end
+
+    # @return [String] A representation suitable for IRB and debugging output
+    def inspect
+      "#<#{self.class.name} #{@bucket.client.http.path(@bucket.client.prefix, @bucket.name, @key).to_s} [#{@content_type}]:#{@data.inspect}>"
+    end
+
+    # Walks links from this object to other objects in Riak.
+    def walk(*params)
+      specs = WalkSpec.normalize(*params)
+      response = @bucket.client.http.get(200, @bucket.client.prefix, @bucket.name, @key, specs.join("/"))
+      if boundary = Multipart.extract_boundary(response[:headers]['content-type'].first)
+        Multipart.parse(response[:body], boundary).map do |group|
+          map_walk_group(group)
+        end
+      else
+        []
+      end
+    end
+    
+    # Converts the object to a link suitable for linking other objects to it
+    def to_link(tag=nil)
+      Link.new(@bucket.client.http.path(@bucket.client.prefix, @bucket.name, @key).path, tag)
+    end
+    
+    private
+    def extract_header(response, name, attribute=nil)
+      if response[:headers][name].present?
+        value = response[:headers][name].try(:first)
+        value = yield value if block_given?
+        send("#{attribute}=", value) if attribute
+      end
+    end
+
+    def map_walk_group(group)
+      group.map do |obj|
+        if obj[:headers] && obj[:body] && obj[:headers]['location']
+          bucket, key = $1, $2 if obj[:headers]['location'].first =~ %r{/.*/(.*)/(.*)$}
+          RObject.new(@bucket.client.bucket(bucket, :keys => false), key).load(obj)
+        end
+      end
+    end
+  end
+end