better-riak-client 1.0.5

data/lib/riak/node/defaults.rb

@@ -0,0 +1,83 @@
+require 'riak/core_ext/deep_dup'
+
+module Riak
+  class Node
+    # Settings based on Riak 1.1.
+    ENV_DEFAULTS = {
+      :riak_core => {
+        :ring_creation_size => 64
+      },
+      :riak_kv => {
+        :storage_backend => :riak_kv_bitcask_backend,
+        :map_js_vm_count => 8,
+        :reduce_js_vm_count => 6,
+        :hook_js_vm_count => 2,
+        :mapper_batch_size => 5,
+        :js_max_vm_mem => 8,
+        :js_thread_stack => 16,
+        :riak_kv_stat => true,
+        :legacy_stats => true,
+        :vnode_vclocks => true,
+        :http_url_encoding => :on,
+        :legacy_keylisting => false,
+        :mapred_system => :pipe,
+        :mapred_2i_pipe => true,
+        :listkeys_backpressure => true,
+        :add_paths => []
+      },
+      :riak_search => {
+        :enabled => true
+      },
+      :luwak => {
+        :enabled => true
+      },
+      :merge_index => {
+        :buffer_rollover_size => 1048576,
+        :max_compact_segments => 20
+      },
+      :eleveldb => {},
+      :bitcask => {},
+      :lager => {
+        :crash_log_size => 10485760,
+        :crash_log_msg_size => 65536,
+        :crash_log_date => "$D0",
+        :crash_log_count => 5,
+        :error_logger_redirect => true
+      },
+      :riak_sysmon => {
+        :process_limit => 30,
+        :port_limit => 30,
+        :gc_ms_limit => 100,
+        :heap_word_limit => 40111000,
+        :busy_port => true,
+        :busy_dist_port => true
+      },
+      :sasl => {
+        :sasl_error_logger => false
+      },
+      :riak_control => {
+        :enabled => false,
+        :auth => :userlist,
+        :userlist => {"user" => "pass"},
+        :admin => true
+      }
+    }.freeze
+
+    # Based on Riak 1.1.
+    VM_DEFAULTS = {
+      "+K" => true,
+      "+A" => 64,
+      "-smp" => "enable",
+      "+W" => "w",
+      "-env ERL_MAX_PORTS" => 4096,
+      "-env ERL_FULLSWEEP_AFTER" => 0
+    }.freeze
+
+    protected
+    # Populates the defaults
+    def set_defaults
+      @env = ENV_DEFAULTS.deep_dup
+      @vm = VM_DEFAULTS.deep_dup
+    end
+  end
+end

data/lib/riak/node/generation.rb

@@ -0,0 +1,106 @@
+require 'yaml'
+
+module Riak
+  class Node
+    # Does the node exist on disk?
+    def exist?
+      manifest.exist?
+    end
+
+    # Deletes the node and regenerates it.
+    def recreate
+      delete
+      create
+    end
+
+    # Generates the node.
+    def create
+      unless exist?
+        create_directories
+        write_scripts
+        write_vm_args
+        write_app_config
+        write_manifest
+      end
+    end
+
+    # Clears data from known data directories. Stops the node if it is
+    # running.
+    def drop
+      was_started = started?
+      stop if was_started
+      data.children.each do |item|
+        if item.directory?
+          item.children.each {|c| c.rmtree }
+        else
+          item.delete
+        end
+      end
+      start if was_started
+    end
+
+    # Removes the node from disk and freezes the object.
+    def destroy
+      delete
+      freeze
+    end
+
+    protected
+    def delete
+      stop unless stopped?
+      root.rmtree if root.exist?
+    end
+
+    def create_directories
+      root.mkpath
+      NODE_DIRECTORIES.each {|d| send(d).mkpath }
+    end
+
+    def write_vm_args
+      (etc + 'vm.args').open('w') do |f|
+        vm.each do |k,v|
+          f.puts "#{k} #{v}"
+        end
+      end
+    end
+
+    def write_app_config
+      (etc + 'app.config').open('w') do |f|
+        f.write to_erlang_config(env) + '.'
+      end
+    end
+
+    def write_scripts
+      [control_script, admin_script].each {|s| write_script(s.basename, s) }
+    end
+
+    def write_script(name, target)
+      source_script = source + name
+      target.open('wb') do |f|
+        source_script.readlines.each do |line|
+          line.sub!(/(RUNNER_SCRIPT_DIR=)(.*)/, '\1' + bin.to_s)
+          line.sub!(/(RUNNER_ETC_DIR=)(.*)/, '\1' + etc.to_s)
+          line.sub!(/(RUNNER_USER=)(.*)/, '\1')
+          line.sub!(/(RUNNER_LOG_DIR=)(.*)/, '\1' + log.to_s)
+          line.sub!(/(PIPE_DIR=)(.*)/, '\1' + pipe.to_s + "/") # PIPE_DIR must have a trailing slash
+          line.sub!(/(PLATFORM_DATA_DIR=)(.*)/, '\1' + data.to_s)
+          line.sub!('grep "$RUNNER_BASE_DIR/.*/[b]eam"', 'grep "$RUNNER_ETC_DIR/app.config"')
+          if line.strip == "RUNNER_BASE_DIR=${RUNNER_SCRIPT_DIR%/*}"
+            line = "RUNNER_BASE_DIR=#{source.parent.to_s}\n"
+          end
+          f.write line
+        end
+      end
+      target.chmod 0755
+    end
+
+    def write_manifest
+      # TODO: For now this only saves the information that was used when
+      # configuring the node. Later we'll verify/warn if the settings
+      # used differ on subsequent generations.
+      @configuration[:env] = @env
+      @configuration[:vm] = @vm
+      manifest.open('w') {|f| YAML.dump(@configuration, f) }
+    end
+  end
+end

data/lib/riak/node/log.rb

@@ -0,0 +1,34 @@
+module Riak
+  class Node
+    LAGER_LEVELS = [
+                    :debug,
+                    :info,
+                    :notice,
+                    :warning,
+                    :error,
+                    :critical,
+                    :alert,
+                    :emergency
+                   ]
+
+    def read_console_log(*levels)
+      console_log = log + 'console.log'
+      if console_log.exist?
+        levels = levels.map { |level| expand_log_level(level) }.compact.flatten
+        pattern = /(#{levels.map { |level| "\\[#{level}\\]" }.join("|")})/
+        console_log.readlines.grep(pattern)
+      end
+    end
+
+    def expand_log_level(level)
+      case level
+      when Range
+        first = LAGER_LEVELS.index(level.begin.to_sym) || 0
+        last = LAGER_LEVELS.index(level.end.to_sym) || -1
+        LAGER_LEVELS[first..last]
+      when Symbol
+        level
+      end
+    end
+  end
+end

data/lib/riak/node/version.rb

@@ -0,0 +1,43 @@
+require 'pathname'
+
+module Riak
+  class Node
+    # @return [String] the version of the Riak node
+    def version
+      @version ||= configure_version
+    end
+
+    # @return [Pathname] the location of Riak installation, aka RUNNER_BASE_DIR
+    def base_dir
+      @base_dir ||= configure_base_dir
+    end
+
+    protected
+    # Detects the Riak version from the generated release
+    def configure_version
+      if base_dir
+        versions = (base_dir + 'releases' + 'start_erl.data').read
+        versions.split(" ")[1]
+      end
+    end
+
+    # Determines the base_dir from source control script
+    def configure_base_dir
+      # Use the script from the source directory so we don't require
+      # it to be generated first.
+      (source + control_script_name).each_line.find {|l| l =~ /^RUNNER_BASE_DIR=(.*)/ }
+
+      # There should only be one matching line, so the contents of $1
+      # will be the matched path. If there's nothing matched, we
+      # return nil.
+      case $1
+      when '${RUNNER_SCRIPT_DIR%/*}'
+        source.parent
+      when String
+        Pathname.new($1).expand_path
+      else
+        nil
+      end
+    end
+  end
+end

data/lib/riak/node.rb

@@ -0,0 +1,38 @@
+require 'riak/util/translation'
+require 'riak/node/defaults'
+require 'riak/node/configuration'
+require 'riak/node/generation'
+require 'riak/node/control'
+require 'riak/node/version'
+require 'riak/node/log'
+
+module Riak
+  # A Node encapsulates the generation and management of standalone
+  # Riak nodes. It is used by the {TestServer} to start and manage an
+  # in-memory node for supporting integration test suites.
+  class Node
+    include Util::Translation
+
+    # Creates a new Riak node. Unlike {#new}, this will also generate
+    # the node if it does not exist on disk.  Equivalent to {::new}
+    # followed by {#create}.
+    # @see #new
+    def self.create(configuration={})
+      new(configuration).tap do |node|
+        node.create
+      end
+    end
+
+    # Creates the template for a Riak node. To generate the node after
+    # initialization, use {#create}.
+    def initialize(configuration={})
+      set_defaults
+      configure configuration
+    end
+
+    protected
+    def debug(msg)
+      $stderr.puts msg if ENV["DEBUG_RIAK_NODE"]
+    end
+  end
+end

data/lib/riak/robject.rb

@@ -0,0 +1,318 @@
+require 'set'
+require 'time'
+require 'yaml'
+require 'riak/util/translation'
+require 'riak/util/escape'
+require 'riak/bucket'
+require 'riak/link'
+require 'riak/walk_spec'
+require 'riak/serializers'
+
+module Riak
+  # Represents the data and metadata stored in a bucket/key pair in
+  # the Riak database, the base unit of data manipulation.
+  class RObject
+    include Util::Translation
+    extend  Util::Translation
+    include Util::Escape
+    extend Util::Escape
+
+    # @return [Bucket] the bucket in which this object is contained
+    attr_accessor :bucket
+
+    # @return [String] the key of this object within its bucket
+    attr_accessor :key
+
+    # @return [String] the MIME content type of the object
+    attr_accessor :content_type
+
+    # @return [String] the Riak vector clock for the object
+    attr_accessor :vclock
+
+    # @return [Set<Link>] a Set of {Riak::Link} objects for relationships between this object and other resources
+    attr_accessor :links
+
+    # @return [String] the ETag header from the most recent HTTP response, useful for caching and reloading
+    attr_accessor :etag
+
+    # @return [Time] the Last-Modified header from the most recent HTTP response, useful for caching and reloading
+    attr_accessor :last_modified
+
+    # @return [Hash] a hash of any X-Riak-Meta-* headers that were in the HTTP response, keyed on the trailing portion
+    attr_accessor :meta
+
+    # @return [Hash<Set>] a hash of secondary indexes, where the
+    #   key is the index name and the value is a Set of index
+    #   entries for that index
+    attr_accessor :indexes
+
+    # @return [Boolean] whether to attempt to prevent stale writes using conditional PUT semantics, If-None-Match: * or If-Match: {#etag}
+    # @see http://wiki.basho.com/display/RIAK/REST+API#RESTAPI-Storeaneworexistingobjectwithakey Riak Rest API Docs
+    attr_accessor :prevent_stale_writes
+
+    # Defines a callback to be invoked when there is conflict.
+    #
+    # @yield The conflict callback.
+    # @yieldparam [RObject] robject The conflicted RObject
+    # @yieldreturn [RObject, nil] Either the resolved RObject or nil if your
+    #                             callback cannot resolve it. The next registered
+    #                             callback will be given the chance to resolve it.
+    #
+    # @note Ripple registers its own document-level conflict handler, so if you're
+    #       using ripple, you will probably want to use that instead.
+    def self.on_conflict(&conflict_hook)
+      on_conflict_hooks << conflict_hook
+    end
+
+    # @return [Array<Proc>] the list of registered conflict callbacks.
+    def self.on_conflict_hooks
+      @on_conflict_hooks ||= []
+    end
+
+    # Attempts to resolve conflict using the registered conflict callbacks.
+    #
+    # @return [RObject] the RObject
+    # @note There is no guarantee the returned RObject will have been resolved
+    def attempt_conflict_resolution
+      return self unless conflict?
+
+      self.class.on_conflict_hooks.each do |hook|
+        result = hook.call(self)
+        return result if result.is_a?(RObject)
+      end
+
+      self
+    end
+
+    # Loads a list of RObjects that were emitted from a MapReduce
+    # query.
+    # @param [Client] client A Riak::Client with which the results will be associated
+    # @param [Array<Hash>] response A list of results a MapReduce job. Each entry should contain these keys: bucket, key, vclock, values
+    # @return [Array<RObject>] An array of RObject instances
+    def self.load_from_mapreduce(client, response)
+      response.map do |item|
+        RObject.new(client[unescape(item['bucket'])], unescape(item['key'])).load_from_mapreduce(item)
+      end
+    end
+
+    # Create a new object manually
+    # @param [Bucket] bucket the bucket in which the object exists
+    # @param [String] key the key at which the object resides. If nil, a key will be assigned when the object is saved.
+    # @yield self the new RObject
+    # @see Bucket#get
+    def initialize(bucket, key=nil)
+      @bucket, @key = bucket, key
+      @links, @meta = Set.new, {}
+      @indexes = new_index_hash
+      yield self if block_given?
+    end
+
+    def indexes=(hash)
+      @indexes = hash.inject(new_index_hash) do |h, (k,v)|
+        h[k].merge([*v])
+        h
+      end
+    end
+
+    # Load object data from a map/reduce response item.
+    # This method is used by RObject::load_from_mapreduce to instantiate the necessary
+    # objects.
+    # @param [Hash] response a response from {Riak::MapReduce}
+    # @return [RObject] self
+    def load_from_mapreduce(response)
+      self.vclock = response['vclock']
+      if response['values'].size == 1
+        value = response['values'].first
+        load_map_reduce_value(value)
+      else
+        @conflict = true
+        @siblings = response['values'].map do |v|
+          RObject.new(self.bucket, self.key) do |robj|
+            robj.vclock = self.vclock
+            robj.load_map_reduce_value(v)
+          end
+        end
+      end
+      self
+    end
+
+    # @return [Object] the unmarshaled form of {#raw_data} stored in riak at this object's key
+    def data
+      if @raw_data && !@data
+        raw = @raw_data.respond_to?(:read) ? @raw_data.read : @raw_data
+        @data = deserialize(raw)
+        @raw_data = nil
+      end
+      @data
+    end
+
+    # @param [Object] unmarshaled form of the data to be stored in riak. Object will be serialized using {#serialize} if a known content_type is used. Setting this overrides values stored with {#raw_data=}
+    # @return [Object] the object stored
+    def data=(new_data)
+      if new_data.respond_to?(:read)
+        raise ArgumentError.new(t("invalid_io_object"))
+      end
+
+      @raw_data = nil
+      @data = new_data
+    end
+
+    # @return [String] raw data stored in riak for this object's key
+    def raw_data
+      if @data && !@raw_data
+        @raw_data = serialize(@data)
+        @data = nil
+      end
+      @raw_data
+    end
+
+    # @param [String, IO-like] the raw data to be stored in riak at this key, will not be marshaled or manipulated prior to storage. Overrides any data stored by {#data=}
+    # @return [String] the data stored
+    def raw_data=(new_raw_data)
+      @data = nil
+      @raw_data = new_raw_data
+    end
+
+    # Store the object in Riak
+    # @param [Hash] options query parameters
+    # @option options [Fixnum] :r the "r" parameter (Read quorum for the implicit read performed when validating the store operation)
+    # @option options [Fixnum] :w the "w" parameter (Write quorum)
+    # @option options [Fixnum] :dw the "dw" parameter (Durable-write quorum)
+    # @option options [Boolean] :returnbody (true) whether to return the result of a successful write in the body of the response. Set to false for fire-and-forget updates, set to true to immediately have access to the object's stored representation.
+    # @return [Riak::RObject] self
+    # @raise [ArgumentError] if the content_type is not defined
+    def store(options={})
+      raise ArgumentError, t("content_type_undefined") unless @content_type.present?
+      @bucket.client.store_object(self, options)
+      self
+    end
+
+    # Reload the object from Riak.  Will use conditional GETs when possible.
+    # @param [Hash] options query parameters
+    # @option options [Fixnum] :r the "r" parameter (Read quorum)
+    # @option options [Boolean] :force will force a reload request if
+    #     the vclock is not present, useful for reloading the object after
+    #     a store (not passed in the query params)
+    # @return [Riak::RObject] self
+    def reload(options={})
+      force = options.delete(:force)
+      return self unless @key && (@vclock || force)
+      self.etag = self.last_modified = nil if force
+      bucket.client.reload_object(self, options)
+    end
+
+    alias :fetch :reload
+
+    # Delete the object from Riak and freeze this instance.  Will work whether or not the object actually
+    # exists in the Riak database.
+    # @see Bucket#delete
+    def delete(options={})
+      return if key.blank?
+      options[:vclock] = vclock if vclock
+      @bucket.delete(key, options)
+      freeze
+    end
+
+    attr_writer :siblings, :conflict
+
+    # Returns sibling objects when in conflict.
+    # @return [Array<RObject>] an array of conflicting sibling objects for this key
+    # @return [Array<self>] a single-element array containing object when not
+    # in conflict
+    def siblings
+      return [self] unless conflict?
+      @siblings
+    end
+
+    # @return [true,false] Whether this object has conflicting sibling objects (divergent vclocks)
+    def conflict?
+      @conflict.present?
+    end
+
+    # Serializes the internal object data for sending to Riak. Differs based on the content-type.
+    # This method is called internally when storing the object.
+    # Automatically serialized formats:
+    # * JSON (application/json)
+    # * YAML (text/yaml)
+    # * Marshal (application/x-ruby-marshal)
+    # When given an IO-like object (e.g. File), no serialization will
+    # be done.
+    # @param [Object] payload the data to serialize
+    def serialize(payload)
+      Serializers.serialize(@content_type, payload)
+    end
+
+    # Deserializes the internal object data from a Riak response. Differs based on the content-type.
+    # This method is called internally when loading the object.
+    # Automatically deserialized formats:
+    # * JSON (application/json)
+    # * YAML (text/yaml)
+    # * Marshal (application/x-ruby-marshal)
+    # @param [String] body the serialized response body
+    def deserialize(body)
+      Serializers.deserialize(@content_type, body)
+    end
+
+    # @return [String] A representation suitable for IRB and debugging output
+    def inspect
+      body = if @data || Serializers[content_type]
+               data.inspect
+             else
+               @raw_data && "(#{@raw_data.size} bytes)"
+             end
+      "#<#{self.class.name} {#{bucket.name}#{"," + @key if @key}} [#{@content_type}]:#{body}>"
+    end
+
+    # Walks links from this object to other objects in Riak.
+    # @param [Array<Hash,WalkSpec>] link specifications for the query
+    def walk(*params)
+      specs = WalkSpec.normalize(*params)
+      @bucket.client.link_walk(self, specs)
+    end
+
+    # Converts the object to a link suitable for linking other objects
+    # to it
+    # @param [String] tag the tag to apply to the link
+    def to_link(tag)
+      Link.new(@bucket.name, @key, tag)
+    end
+
+    alias :vector_clock :vclock
+    alias :vector_clock= :vclock=
+
+    protected
+    def load_map_reduce_value(hash)
+      metadata = hash['metadata']
+      extract_if_present(metadata, 'X-Riak-VTag', :etag)
+      extract_if_present(metadata, 'content-type', :content_type)
+      extract_if_present(metadata, 'X-Riak-Last-Modified', :last_modified) { |v| Time.httpdate( v ) }
+      extract_if_present(metadata, 'index', :indexes) do |entries|
+        Hash[ entries.map {|k,v| [k, Set.new(Array(v))] } ]
+      end
+      extract_if_present(metadata, 'Links', :links) do |links|
+        Set.new( links.map { |l| Link.new(*l) } )
+      end
+      extract_if_present(metadata, 'X-Riak-Meta', :meta) do |meta|
+        Hash[
+             meta.map do |k,v|
+               [k.sub(%r{^x-riak-meta-}i, ''), [v]]
+             end
+            ]
+      end
+      extract_if_present(hash, 'data', :data) { |v| deserialize(v) }
+    end
+
+    private
+    def extract_if_present(hash, key, attribute=nil)
+      if hash[key].present?
+        attribute ||= key
+        value = block_given? ? yield(hash[key]) : hash[key]
+        send("#{attribute}=", value)
+      end
+    end
+
+    def new_index_hash
+      Hash.new {|h,k| h[k] = Set.new }
+    end
+  end
+end