mongo 1.0 → 1.1.5

data/lib/mongo/gridfs/grid_io.rb

@@ -1,3 +1,5 @@
+# encoding: UTF-8
+
 # --
 # Copyright (C) 2008-2010 10gen Inc.
 #
@@ -30,7 +32,7 @@ module Mongo
     PROTECTED_ATTRS      = [:files_id, :file_length, :client_md5, :server_md5]
 
     attr_reader :content_type, :chunk_size, :upload_date, :files_id, :filename,
-      :metadata, :server_md5, :client_md5, :file_length
+      :metadata, :server_md5, :client_md5, :file_length, :file_position
 
     # Create a new GridIO object. Note that most users will not need to use this class directly;
     # the Grid and GridFileSystem classes will instantiate this class
@@ -43,14 +45,14 @@ module Mongo
     # @option opts [Hash] :query a query selector used when opening the file in 'r' mode.
     # @option opts [Hash] :query_opts any query options to be used when opening the file in 'r' mode.
     # @option opts [String] :fs_name the file system prefix.
-    # @options opts [Integer] (262144) :chunk_size size of file chunks in bytes.
-    # @options opts [Hash] :metadata ({}) any additional data to store with the file.
-    # @options opts [ObjectID] :_id (ObjectID) a unique id for
+    # @option opts [Integer] (262144) :chunk_size size of file chunks in bytes.
+    # @option opts [Hash] :metadata ({}) any additional data to store with the file.
+    # @option opts [ObjectId] :_id (ObjectId) a unique id for
     #   the file to be use in lieu of an automatically generated one.
-    # @options opts [String] :content_type ('binary/octet-stream') If no content type is specified,
+    # @option opts [String] :content_type ('binary/octet-stream') If no content type is specified,
     #   the content type will may be inferred from the filename extension if the mime-types gem can be
     #   loaded. Otherwise, the content type 'binary/octet-stream' will be used.
-    # @options opts [Boolean] :safe (false) When safe mode is enabled, the chunks sent to the server
+    # @option opts [Boolean] :safe (false) When safe mode is enabled, the chunks sent to the server
     #   will be validated using an md5 hash. If validation fails, an exception will be raised.
     def initialize(files, chunks, filename, mode, opts={})
       @files        = files
@@ -178,7 +180,7 @@ module Mongo
     # This method will be invoked automatically when
     # on GridIO#open is passed a block. Otherwise, it must be called manually.
     #
-    # @return [BSON::ObjectID]
+    # @return [BSON::ObjectId]
     def close
       if @mode[0] == ?w
         if @current_chunk['n'].zero? && @chunk_position.zero?
@@ -190,6 +192,23 @@ module Mongo
       id
     end
 
+    # Read a chunk of the data from the file and yield it to the given
+    # block.
+    #
+    # Note that this method reads from the current file position.
+    #
+    # @yield Yields on chunk per iteration as defined by this file's
+    #   chunk size.
+    #
+    # @return [Mongo::GridIO] self
+    def each
+      return read_all unless block_given?
+      while chunk = read(chunk_size)
+        yield chunk
+      end
+      self
+    end
+
     def inspect
       "#<GridIO _id: #{@files_id}>"
     end
@@ -197,8 +216,8 @@ module Mongo
     private
 
     def create_chunk(n)
-      chunk = OrderedHash.new
-      chunk['_id']      = BSON::ObjectID.new
+      chunk = BSON::OrderedHash.new
+      chunk['_id']      = BSON::ObjectId.new
       chunk['n']        = n
       chunk['files_id'] = @files_id
       chunk['data']     = ''
@@ -223,11 +242,14 @@ module Mongo
     # Read a file in its entirety.
     def read_all
       buf = ''
-      while true
+      if @current_chunk
         buf << @current_chunk['data'].to_s
-        @current_chunk = get_chunk(@current_chunk['n'] + 1)
-        break unless @current_chunk
+        while chunk = get_chunk(@current_chunk['n'] + 1)
+          buf << chunk['data'].to_s
+          @current_chunk = chunk
+        end
       end
+      @file_position = @file_length
       buf
     end
 
@@ -235,7 +257,11 @@ module Mongo
     def read_length(length)
       cache_chunk_data
       remaining  = (@file_length - @file_position)
-      to_read    = length > remaining ? remaining : length
+      if length.nil?
+        to_read = remaining
+      else
+        to_read    = length > remaining ? remaining : length
+      end
       return nil unless remaining > 0
 
       buf        =  ''
@@ -306,20 +332,27 @@ module Mongo
 
     # Initialize the class for writing a file.
     def init_write(opts)
-      @files_id      = opts.delete(:_id) || BSON::ObjectID.new
+      @files_id      = opts.delete(:_id) || BSON::ObjectId.new
       @content_type  = opts.delete(:content_type) || (defined? MIME) && get_content_type || DEFAULT_CONTENT_TYPE
       @chunk_size    = opts.delete(:chunk_size) || DEFAULT_CHUNK_SIZE
       @metadata      = opts.delete(:metadata) if opts[:metadata]
       @aliases       = opts.delete(:aliases) if opts[:aliases]
       @file_length   = 0
       opts.each {|k, v| self[k] = v}
+      check_existing_file if @safe
 
       @current_chunk = create_chunk(0)
       @file_position = 0
     end
 
+    def check_existing_file
+      if @files.find_one('_id' => @files_id)
+        raise GridError, "Attempting to overwrite with Grid#put. You must delete the file first."
+      end
+    end
+
     def to_mongo_object
-      h                = OrderedHash.new
+      h                = BSON::OrderedHash.new
       h['_id']         = @files_id
       h['filename']    = @filename if @filename
       h['contentType'] = @content_type
@@ -335,7 +368,7 @@ module Mongo
 
     # Get a server-side md5 and validate against the client if running in safe mode.
     def get_md5
-      md5_command            = OrderedHash.new
+      md5_command            = BSON::OrderedHash.new
       md5_command['filemd5'] = @files_id
       md5_command['root']    = @fs_name
       @server_md5 = @files.db.command(md5_command)['md5']

data/lib/mongo/gridfs/grid_io_fix.rb

@@ -0,0 +1,38 @@
+# encoding: UTF-8
+
+# --
+# Copyright (C) 2008-2010 10gen Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ++
+
+module Mongo
+  class GridIO
+
+    # This fixes a comparson issue in JRuby 1.9
+    def get_md5
+      md5_command            = BSON::OrderedHash.new
+      md5_command['filemd5'] = @files_id
+      md5_command['root']    = @fs_name
+      @server_md5 = @files.db.command(md5_command)['md5']
+      if @safe
+        @client_md5 = @local_md5.hexdigest
+        if @local_md5.to_s != @server_md5.to_s
+          raise GridMD5Failure, "File on server failed MD5 check"
+        end
+      else
+        @server_md5
+      end
+    end
+  end
+end

data/lib/mongo/repl_set_connection.rb

@@ -0,0 +1,290 @@
+# encoding: UTF-8
+
+# --
+# Copyright (C) 2008-2010 10gen Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ++
+
+module Mongo
+
+  # Instantiates and manages connections to a MongoDB replica set.
+  class ReplSetConnection < Connection
+    attr_reader :nodes, :secondaries, :arbiters, :read_pool, :secondary_pools
+
+    # Create a connection to a MongoDB replica set.
+    #
+    # Once connected to a replica set, you can find out which nodes are primary, secondary, and
+    # arbiters with the corresponding accessors: Connection#primary, Connection#secondaries, and
+    # Connection#arbiters. This is useful if your application needs to connect manually to nodes other
+    # than the primary.
+    #
+    # @param [Array] args A list of host-port pairs ending with a hash containing any options. See
+    #   the examples below for exactly how to use the constructor.
+    #
+    # @option options [Boolean, Hash] :safe (false) Set the default safe-mode options
+    #   propogated to DB objects instantiated off of this Connection. This
+    #   default can be overridden upon instantiation of any DB by explicity setting a :safe value
+    #   on initialization.
+    # @option options [Boolean] :read_secondary(false) If true, a random secondary node will be chosen,
+    #   and all reads will be directed to that node.
+    # @option options [Logger, #debug] :logger (nil) Logger instance to receive driver operation log.
+    # @option options [Integer] :pool_size (1) The maximum number of socket connections allowed per
+    #   connection pool. Note: this setting is relevant only for multi-threaded applications.
+    # @option options [Float] :timeout (5.0) When all of the connections a pool are checked out,
+    #   this is the number of seconds to wait for a new connection to be released before throwing an exception.
+    #   Note: this setting is relevant only for multi-threaded applications.
+    #
+    # @example Connect to a replica set and provide two seed nodes:
+    #   ReplSetConnection.new(['localhost', 30000], ['localhost', 30001])
+    #
+    # @example Connect to a replica set providing two seed nodes and allowing reads from a
+    #   secondary node:
+    #   ReplSetConnection.new(['localhost', 30000], ['localhost', 30001], :read_secondary => true)
+    #
+    # @see http://api.mongodb.org/ruby/current/file.REPLICA_SETS.html Replica sets in Ruby
+    #
+    # @raise [ReplicaSetConnectionError] This is raised if a replica set name is specified and the
+    #   driver fails to connect to a replica set with that name.
+    def initialize(*args)
+      if args.last.is_a?(Hash)
+        opts = args.pop
+      else
+        opts = {}
+      end
+
+      unless args.length > 0
+        raise MongoArgumentError, "A ReplSetConnection requires at least one node."
+      end
+
+      # Get seed nodes
+      @nodes = args
+
+      # Replica set name
+      @replica_set = opts[:rs_name]
+
+      # Cache the various node types when connecting to a replica set.
+      @secondaries = []
+      @arbiters    = []
+
+      # Connection pools for each secondary node
+      @secondary_pools = []
+      @read_pool = nil
+
+      # Are we allowing reads from secondaries?
+      @read_secondary = opts.fetch(:read_secondary, false)
+
+      setup(opts)
+    end
+
+    # Create a new socket and attempt to connect to master.
+    # If successful, sets host and port to master and returns the socket.
+    #
+    # If connecting to a replica set, this method will replace the
+    # initially-provided seed list with any nodes known to the set.
+    #
+    # @raise [ConnectionFailure] if unable to connect to any host or port.
+    def connect
+      reset_connection
+      @nodes_to_try = @nodes.clone
+
+      while connecting?
+        node   = @nodes_to_try.shift
+        config = check_is_master(node)
+
+        if is_primary?(config)
+          set_primary(node)
+        else
+          set_auxillary(node, config)
+        end
+      end
+
+      pick_secondary_for_read if @read_secondary
+
+      if !connected?
+        if @secondary_pools.empty?
+          raise ConnectionFailure, "Failed to connect any given host:port"
+        else
+          raise ConnectionFailure, "Failed to connect to primary node."
+        end
+      end
+    end
+
+    def connecting?
+      @nodes_to_try.length > 0
+    end
+
+    # Close the connection to the database.
+    def close
+      super
+      @read_pool = nil
+      @secondary_pools.each do |pool|
+        pool.close
+      end
+    end
+
+    # If a ConnectionFailure is raised, this method will be called
+    # to close the connection and reset connection values.
+    # TODO: what's the point of this method?
+    def reset_connection
+      super
+      @secondaries     = []
+      @secondary_pools = []
+      @arbiters        = []
+      @nodes_tried  = []
+      @nodes_to_try = []
+    end
+
+    # Is it okay to connect to a slave?
+    #
+    # @return [Boolean]
+    def slave_ok?
+      @read_secondary || @slave_ok
+    end
+
+    private
+
+    def check_is_master(node)
+      begin
+        host, port = *node
+        socket = TCPSocket.new(host, port)
+        socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)
+
+        config = self['admin'].command({:ismaster => 1}, :sock => socket)
+
+        check_set_name(config, socket)
+      rescue OperationFailure, SocketError, SystemCallError, IOError => ex
+        close unless connected?
+      ensure
+        @nodes_tried << node
+        if config
+          nodes = []
+          nodes += config['hosts'] if config['hosts']
+          nodes += config['arbiters'] if config['arbiters']
+          nodes += config['passives'] if config['passives']
+          update_node_list(nodes)
+
+          if config['msg'] && @logger
+            @logger.warn("MONGODB #{config['msg']}")
+          end
+        end
+
+        socket.close if socket
+      end
+
+      config
+    end
+
+    # Primary, when connecting to a replica can, can only be a true primary node.
+    # (And not a slave, which is possible when connecting with the standard
+    # Connection class.
+    def is_primary?(config)
+      config && (config['ismaster'] == 1 || config['ismaster'] == true)
+    end
+
+    # Pick a node randomly from the set of possible secondaries.
+    def pick_secondary_for_read
+      if (size = @secondary_pools.size) > 0
+        @read_pool = @secondary_pools[rand(size)]
+      end
+    end
+
+    # Make sure that we're connected to the expected replica set.
+    def check_set_name(config, socket)
+      if @replica_set
+        config = self['admin'].command({:replSetGetStatus => 1},
+                   :sock => socket, :check_response => false)
+
+        if !Mongo::Support.ok?(config)
+          raise ReplicaSetConnectionError, config['errmsg']
+        elsif config['set'] != @replica_set
+          raise ReplicaSetConnectionError,
+            "Attempting to connect to replica set '#{config['set']}' but expected '#{@replica_set}'"
+        end
+      end
+    end
+
+    # Determines what kind of node we have and caches its host
+    # and port so that users can easily connect manually.
+    def set_auxillary(node, config)
+      if config
+        if config['secondary']
+          host, port = *node
+          @secondaries << node unless @secondaries.include?(node)
+          @secondary_pools << Pool.new(self, host, port, :size => @pool_size, :timeout => @timeout)
+        elsif config['arbiterOnly']
+          @arbiters << node unless @arbiters.include?(node)
+        end
+      end
+    end
+
+    # Update the list of known nodes. Only applies to replica sets,
+    # where the response to the ismaster command will return a list
+    # of known hosts.
+    #
+    # @param hosts [Array] a list of hosts, specified as string-encoded
+    #   host-port values. Example: ["myserver-1.org:27017", "myserver-1.org:27017"]
+    #
+    # @return [Array] the updated list of nodes
+    def update_node_list(hosts)
+      new_nodes = hosts.map do |host|
+        if !host.respond_to?(:split)
+          warn "Could not parse host #{host.inspect}."
+          next
+        end
+
+        host, port = host.split(':')
+        [host, port ? port.to_i : Connection::DEFAULT_PORT]
+      end
+
+      # Replace the list of seed nodes with the canonical list.
+      @nodes = new_nodes.clone
+
+      @nodes_to_try = new_nodes - @nodes_tried
+    end
+
+    # Checkout a socket for reading (i.e., a secondary node).
+    def checkout_reader
+      connect unless connected?
+
+      if @read_pool
+        @read_pool.checkout
+      else
+        checkout_writer
+      end
+    end
+
+    # Checkout a socket for writing (i.e., a primary node).
+    def checkout_writer
+      connect unless connected?
+
+      @primary_pool.checkout
+    end
+
+    # Checkin a socket used for reading.
+    def checkin_reader(socket)
+      if @read_pool
+        @read_pool.checkin(socket)
+      else
+        checkin_writer(socket)
+      end
+    end
+
+    # Checkin a socket used for writing.
+    def checkin_writer(socket)
+      if @primary_pool
+        @primary_pool.checkin(socket)
+      end
+    end
+  end
+end

data/lib/mongo/util/conversions.rb

@@ -1,3 +1,5 @@
+# encoding: UTF-8
+
 # --
 # Copyright (C) 2008-2010 10gen Inc.
 #
@@ -31,7 +33,7 @@ module Mongo #:nodoc:
     # <tt>array_as_sort_parameters([["field1", :asc], ["field2", :desc]])</tt> =>
     # <tt>{ "field1" => 1, "field2" => -1}</tt>
     def array_as_sort_parameters(value)
-      order_by = OrderedHash.new
+      order_by = BSON::OrderedHash.new
       if value.first.is_a? Array
         value.each do |param|
           if (param.class.name == "String")

data/lib/mongo/util/core_ext.rb

@@ -1,3 +1,5 @@
+# encoding: UTF-8
+
 # --
 # Copyright (C) 2008-2010 10gen Inc.
 #
@@ -18,10 +20,21 @@
 class Object
 
   #:nodoc:
-  def returning(value)
-    yield value
-    value
-  end
+  def tap
+    yield self
+    self
+  end unless respond_to? :tap
+
+end
+
+#:nodoc:
+module Enumerable
+
+  #:nodoc:
+  def each_with_object(memo)
+    each { |element| yield(element, memo) }
+    memo
+  end unless [].respond_to?(:each_with_object)
 
 end
 

data/lib/mongo/util/pool.rb

@@ -0,0 +1,125 @@
+# encoding: UTF-8
+
+# --
+# Copyright (C) 2008-2010 10gen Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Mongo
+  class Pool
+
+    attr_accessor :host, :port, :size, :timeout, :safe, :checked_out
+
+    # Create a new pool of connections.
+    #
+    def initialize(connection, host, port, options={})
+      @connection  = connection
+
+      @host, @port = host, port
+
+      # Pool size and timeout.
+      @size      = options[:size] || 1
+      @timeout   = options[:timeout]   || 5.0
+
+      # Mutex for synchronizing pool access
+      @connection_mutex = Mutex.new
+
+      # Condition variable for signal and wait
+      @queue = ConditionVariable.new
+
+      @sockets      = []
+      @checked_out  = []
+    end
+
+    def close
+      @sockets.each do |sock|
+        begin
+          sock.close
+        rescue IOError => ex
+          warn "IOError when attempting to close socket connected "
+           + "to #{@host}:#{@port}: #{ex.inspect}"
+        end
+      end
+      @host = @port = nil
+      @sockets.clear
+      @checked_out.clear
+    end
+
+    # Return a socket to the pool.
+    def checkin(socket)
+      @connection_mutex.synchronize do
+        @checked_out.delete(socket)
+        @queue.signal
+      end
+      true
+    end
+
+    # Adds a new socket to the pool and checks it out.
+    #
+    # This method is called exclusively from #checkout;
+    # therefore, it runs within a mutex.
+    def checkout_new_socket
+      begin
+      socket = TCPSocket.new(@host, @port)
+      socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)
+      rescue => ex
+        raise ConnectionFailure, "Failed to connect socket: #{ex}"
+      end
+      @sockets << socket
+      @checked_out << socket
+      socket
+    end
+
+    # Checks out the first available socket from the pool.
+    #
+    # This method is called exclusively from #checkout;
+    # therefore, it runs within a mutex.
+    def checkout_existing_socket
+      socket = (@sockets - @checked_out).first
+      @checked_out << socket
+      socket
+    end
+
+    # Check out an existing socket or create a new socket if the maximum
+    # pool size has not been exceeded. Otherwise, wait for the next
+    # available socket.
+    def checkout
+      @connection.connect if !@connection.connected?
+      start_time = Time.now
+      loop do
+        if (Time.now - start_time) > @timeout
+            raise ConnectionTimeoutError, "could not obtain connection within " +
+              "#{@timeout} seconds. The max pool size is currently #{@size}; " +
+              "consider increasing the pool size or timeout."
+        end
+
+        @connection_mutex.synchronize do
+          socket = if @checked_out.size < @sockets.size
+                     checkout_existing_socket
+                   elsif @sockets.size < @size
+                     checkout_new_socket
+                   end
+
+          return socket if socket
+
+          # Otherwise, wait
+          if @logger
+            @logger.warn "MONGODB Waiting for available connection; " +
+              "#{@checked_out.size} of #{@size} connections checked out."
+          end
+          @queue.wait(@connection_mutex)
+        end
+      end
+    end
+  end
+end

data/lib/mongo/util/server_version.rb

@@ -1,3 +1,5 @@
+# encoding: UTF-8
+
 # --
 # Copyright (C) 2008-2010 10gen Inc.
 #

data/lib/mongo/util/support.rb

@@ -1,3 +1,5 @@
+# encoding: UTF-8
+
 # --
 # Copyright (C) 2008-2010 10gen Inc.
 #
@@ -66,5 +68,15 @@ module Mongo
             "[['field1', '(ascending|descending)'], ['field2', '(ascending|descending)']]"
       end
     end
+
+    # Determine if a database command has succeeded by
+    # checking the document response.
+    #
+    # @param [Hash] doc
+    #
+    # @return [Boolean] true if the 'ok' key is either 1 or *true*.
+    def ok?(doc)
+      doc['ok'] == 1.0 || doc['ok'] == true
+    end
   end
 end