monga 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,9 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in monga.gemspec
+gemspec
+
+group :test do
+  gem "minitest"
+  gem "minitest-reporters"
+end

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Petr Yanovich
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,110 @@
+# Monga
+
+[MongoDB](http://www.mongodb.org/) Ruby Client on [EventMachine](https://github.com/eventmachine/eventmachine).
+
+This client is under development. You can try [em-mongo](https://github.com/bcg/em-mongo).
+
+Client supports MongoDB 2.4. Some features won't work in lower versions.
+
+## To Do List
+
+* [ ] Write a Wiki
+* [ ] Write comments
+* [ ] Grammar improvement ;)
+
+### Clients
+* [x] Client (Single instance connection)
+* [IN PROCESS] ReplicaSetClient
+* [ ] MasterSlaveClient
+* [ ] ReadPref
+* [ ] Sharding Support
+
+### Connection
+* [x] Connection
+* [x] Autoreconnect
+* [x] Connection Pool
+
+### Protocol
+* [x] OP_QUERY
+* [x] OP_GET_MORE
+* [x] OP_KILL_CURSORS
+* [x] OP_INSERT
+* [x] OP_UPDATE
+* [x] OP_DELETE
+* [x] OP_REPLY
+
+### Database
+* [x] create_collection
+* [x] drop_collection
+* [x] get_last_error
+* [x] drop_indexes
+* [x] get_indexes
+* Authentication
+    * [ ] login
+    * [ ] logout
+    * [ ] add_user
+* [ ] check maxBsonSize / validate
+* [x] cmd
+* [x] eval
+* [ ] where
+
+### Collection
+* QUERY_OP
+    * [x] find
+    * [x] find_one (first)
+    * [ ] sorting
+* INSERT_OP
+    * [x] insert (single)
+    * [x] insert (batch)
+    * [x] safe_insert
+    * FLAGS
+        * [x] continue_on_error
+* UPDATE_OP
+    * [x] update
+    * [x] safe_update
+    * FLAGS
+        * [x] upsert
+        * [x] multi_update
+* DELETE_OP
+    * [x] delete
+    * [x] safe_delete
+    * FLAGS
+        * [x] single_remove
+* INDEXES
+    * [x] ensure_index
+    * [x] drop_index
+    * [x] drop_indexes
+    * [x] get_indexes
+* [x] count
+* [x] all
+* [x] cursor
+* [ ] DBRef
+
+### Cursor
+* [x] limit
+* [x] skip
+* [x] batch_size
+* [x] get_more
+* [x] next_document
+* [x] next_batch
+* [x] each_doc
+* [x] kill
+* [x] mark_to_kill
+* [x] batch_kill
+* [x] explain
+* [x] hint
+* Flags
+    * [x] tailable_cursor
+    * [x] slave_ok
+    * [x] no_cursor_timeout
+    * [x] await_data
+    * [x] exhaust
+    * [x] partial
+
+# ISSUES handled with
+
+Some commands, such as `db.getLastError`, `db.count` and other `db.commands` requires `numberToReturn` in OP_QUERY to be setted as `-1`. Also this commands should return a response. If nothing returned it should be interpreted as an Exception. Also, in contrast to the other queries it could return `errmsg` which should be interpreted as an Exception too. Other query methods could return `err` response.
+
+To create index you can't call any `db.ensureIndex` command but you should insert a document into `sytem.indexes` collection manually. To get list of indexes you should fetch all documents from this collection. But to drop index you should call specific `db.dropIndexes` command.
+
+`multi_update` flag works only with `$` commands (i.e. `$set: { title: "blahblah" }`)

data/Rakefile

@@ -0,0 +1,13 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+task :default => :spec
+Rake::TestTask.new(:spec) do |t|
+  t.libs << 'spec'
+  if spec = ENV['spec']
+    t.pattern = "spec/**/#{spec}*_spec.rb"
+  else
+    t.pattern = 'spec/**/*_spec.rb'
+  end
+  t.verbose = false
+end

data/lib/monga/client.rb

@@ -0,0 +1,8 @@
+require File.expand_path("../clients/client", __FILE__)
+require File.expand_path("../clients/replica_set_client", __FILE__)
+require File.expand_path("../clients/master_slave_client", __FILE__)
+module Monga
+  Client = Monga::Clients::Client
+  ReplicaSetClient = Monga::Clients::ReplicaSetClient
+  MasterSlaveClient = Monga::Clients::MasterSlaveClient
+end

data/lib/monga/clients/client.rb

@@ -0,0 +1,24 @@
+module Monga::Clients
+  class Client
+    extend Forwardable
+
+    def_delegators :@connection_pool, :aquire_connection, :send_command, :primary?, :connected?
+    
+    attr_reader :connection_pool
+
+    def initialize(opts={})
+      opts[:pool_size] ||= 1
+      @connection_pool = Monga::ConnectionPool.new(opts)
+    end
+
+    def [](db_name)
+      Monga::Database.new(self, db_name)
+    end
+
+    def find_primary!
+      @connection_pool.connections.each do |conn|
+        conn.is_master?(self)
+      end
+    end
+  end
+end

data/lib/monga/clients/master_slave_client.rb

@@ -0,0 +1,5 @@
+module Monga::Clients
+  class MasterSlaveClient
+
+  end
+end

data/lib/monga/clients/replica_set_client.rb

@@ -0,0 +1,101 @@
+# How it works
+# Replica Set tries to establish connections to all passed servers.
+# Till no connection established it queues al queries inside it's
+
+module Monga::Clients
+  class ReplicaSetClient
+    class ProxyConnection
+      include EM::Deferrable
+      def initialize(client)
+        @client = client
+      end
+
+      def send_command(msg, request_id=nil, &cb)
+        callback do
+          connection = @client.aquire_connection
+          connection.send_command(msg, request_id, &cb)
+        end
+      end
+    end
+
+    include EM::Deferrable
+
+    attr_reader :servers, :clients
+
+    def initialize(opts = {})
+      @read_pref = opts.delete(:read_pref) || :primary
+      @servers = opts.delete(:servers)
+      raise ArgumentError, "servers option is not passed or empty" if @servers.empty?
+
+      @clients = @servers.map do |server|
+        Monga::Client.new(server.merge(opts))
+      end
+
+      @proxy_connection = ProxyConnection.new(self)
+    end
+
+    def [](db_name)
+      Monga::Database.new(self, db_name)
+    end
+
+    def aquire_connection
+      server ||= case @read_pref
+      when :primary
+        primary
+      when :secondary
+        secondary
+      when :primary_preferred
+        primary || secondary
+      when :secondary_preferred
+        secondary || primary
+      when :nearest
+        fail "unimplemented read_pref mode"
+      else
+        fail "read_pref is undefined"
+      end
+
+
+      if server
+        if @deferred_status != :succeeded
+          set_deferred_status :succeeded 
+          @proxy_connection.set_deferred_status :succeeded
+        end
+      else
+        if @deferred_status == :succeeded
+          set_deferred_status nil
+          @proxy_connection.set_deferred_status nil
+        end
+      end
+
+      server || @proxy_connection
+    end
+
+    def primary
+      prim = @clients.detect{ |c| c.primary? && c.connected? }
+      unless prim
+        find_primary!
+      end
+      prim
+    end
+
+    def secondary
+      @clients.select{ |c| !c.primary? && c.connected? }.sample
+    end
+
+    def find_primary!
+      unless @pending_primary
+        @pending_primary = true
+        @clients.each{ |c| c.find_primary! }
+        EM.add_timer(0.1) do
+          if primary
+            aquire_connection
+            @pending_primary = false
+          else
+            @pending_primary = false
+            find_primary!
+          end
+        end
+      end
+    end
+  end
+end

data/lib/monga/collection.rb

@@ -0,0 +1,108 @@
+module Monga
+  class Collection
+    attr_reader :name, :db
+
+    def initialize(db, name)
+      @db = db
+      @name = name
+    end
+
+    def query(query = {}, fields = {}, opts = {})
+      options = {}
+      options[:query] = query
+      options[:fields] = fields
+      options.merge! opts
+      Monga::Miner.new(db, name, options)
+    end
+    alias :find :query
+
+    def find_one(query = {}, fields = {}, opts = {})
+      options = {}
+      options[:query] = query
+      options[:fields] = fields
+      options.merge! opts
+
+      Monga::Response.surround do |resp|
+        req = Monga::Miner.new(db, name, options).limit(1)
+        req.callback{ |data| resp.succeed data.first }
+        req.errback{ |err| resp.fail err }
+      end
+    end
+    alias :first :find_one
+
+    def insert(documents, opts = {})
+      options = {}
+      options[:documents] = documents
+      options.merge!(opts)
+      Monga::Requests::Insert.new(@db, @name, options).perform
+    end
+
+    def update(query = {}, update = {}, flags = {})
+      options = {}
+      options[:query] = query
+      options[:update] = update
+      options.merge!(flags)
+      Monga::Requests::Update.new(@db, @name, options).perform
+    end
+
+    def delete(query = {}, opts = {})
+      options = {}
+      options[:query] = query
+      options.merge!(opts)
+      Monga::Requests::Delete.new(@db, @name, options).perform
+    end
+    alias :remove :delete
+
+    def ensure_index(keys, opts={})
+      doc = {}
+      doc.merge!(opts)
+      # Read docs about naming
+      doc[:name] ||= keys.to_a.flatten * "_"
+      doc[:key] = keys
+      doc[:ns] = "#{db.name}.#{name}"
+      Monga::Requests::Insert.new(@db, "system.indexes", {documents: doc}).perform
+    end
+
+    def drop_index(indexes)
+      @db.drop_indexes(@name, indexes)
+    end
+
+    def drop_indexes
+      @db.drop_indexes(@name, "*")
+    end
+
+    def get_indexes
+      Monga::Miner.new(@db, "system.indexes")
+    end
+
+    def drop
+      @db.drop_collection(@name)
+    end
+
+    def count
+      @db.count(@name)
+    end
+
+    # Safe methods
+    [:update, :insert, :delete, :remove, :ensure_index].each do |meth|
+      class_eval <<-EOS
+        def safe_#{meth}(*args)
+          safe do
+            #{meth}(*args)
+          end
+        end
+      EOS
+    end
+
+    def safe
+      response = Monga::Response.new
+      request_id = yield
+      req = @db.get_last_error
+      req.callback do |data|
+        response.succeed(request_id)
+      end
+      req.errback{ |err| response.fail(err) }
+      response
+    end
+  end
+end

data/lib/monga/connection.rb

@@ -0,0 +1,23 @@
+require File.expand_path("../connections/em_connection", __FILE__)
+require File.expand_path("../connections/primary", __FILE__)
+require File.expand_path("../connections/secondary", __FILE__)
+
+module Monga
+  class Connection
+    extend Forwardable
+
+    def_delegators :@connection, :connected?, :reconnect, :responses, :send_command, :master?, :is_master?, :host, :port
+
+    CONNECTION_TYPES = {
+      default: Monga::Connections::EMConnection,
+      primary: Monga::Connections::Primary,
+      secondary: Monga::Connections::Secondary,
+    }
+
+    def initialize(opts={})
+      conn_type = opts.delete(:connection_type) || :default
+      conn_class = CONNECTION_TYPES[conn_type]
+      @connection = conn_class.connect(opts)
+    end
+  end
+end

data/lib/monga/connection_pool.rb

@@ -0,0 +1,38 @@
+module Monga
+  class ConnectionPool < Monga::Connection
+    extend Forwardable
+
+    def_delegators :aquire_connection, :send_command
+
+    attr_reader :connections
+
+    def initialize(opts={})
+      @connections = []
+      pool_size = opts.delete :pool_size
+
+      pool_size.times do
+        @connections << Monga::Connection.new(opts)
+      end
+    end
+
+    def aquire_connection
+      connected = @connections.select(&:connected?)
+      if connected.any?
+        min = connected.min_by{ |c| c.responses.size }.responses.size
+        conns = connected.select{ |c| c.responses.size == min }
+        conns.sample
+      else
+        @connections.sample
+      end
+    end
+
+    def primary?
+      conn = aquire_connection
+      conn ? conn.master? : false
+    end
+
+    def connected?
+      @connections.any?(&:connected?)
+    end
+  end
+end

data/lib/monga/connections/em_connection.rb

@@ -0,0 +1,152 @@
+module Monga::Connections
+  class EMConnection < EM::Connection
+    include EM::Deferrable
+
+    class Buffer
+      include Enumerable
+
+      attr_reader :buffer
+
+      def initialize
+        @buffer = ""
+        @positon = 0
+      end
+
+      def append(data)
+        @buffer += data
+      end
+
+      def each
+        while true
+          size = @buffer.size
+          if size > Monga::HEADER_SIZE
+            msg_length = @buffer[0, 4].unpack("L").first
+            if msg_length && size >= msg_length
+              data = @buffer.slice!(0, size)
+              yield data.unpack("LLLLLQLLa*")
+            else
+              break
+            end
+          else
+            break
+          end
+        end
+      end
+    end
+
+    attr_reader :responses, :host, :port
+
+    def initialize(opts = {})
+      @host = opts[:host]
+      @port = opts[:port]
+      @reactor_running = true
+      @responses = {}
+    end
+
+    def self.connect(opts = {})
+      host = opts[:host] ||= Monga::DEFAULT_HOST
+      port = opts[:port] ||= Monga::DEFAULT_PORT
+
+      EM.connect(host, port, self, opts)
+    end
+
+    def send_command(msg, request_id=nil, &cb)
+      reconnect unless @connected
+
+      callback do
+        send_data msg
+      end
+
+      @responses[request_id] = cb if cb
+    end
+
+    def receive_data(data)
+      @buffer.append(data)
+      @buffer.each do |message|
+        request_id = message[2]
+        cb = @responses.delete request_id
+        cb.call(message) if cb
+      end
+    end
+
+    def connection_completed
+      Monga.logger.debug("Connection is established #{@host}:#{@port}")
+
+      EM.add_shutdown_hook do
+        close
+      end
+
+      unless @reactor_running
+        EM.add_periodic_timer(Monga::Cursor::CLOSE_TIMEOUT){ Monga::Cursor.batch_kill(self) }
+      end
+
+      @connected = true
+      @pending_for_reconnect = false
+      @buffer = Buffer.new
+      @reactor_running = true
+
+      succeed
+    end
+
+    def reconnect
+      unless @connected && @pending_for_reconnect
+        if @reactor_running
+          super(@host, @port)
+        else
+          EM.schedule{ super(@host, @port) }
+        end
+        @pending_for_reconnect = true
+      end
+    end
+
+    def force_reconnect(host, port)
+      @connected = false
+      @host = host
+      @port = port
+    end
+
+    def connected?
+      EM.schedule { reconnect } unless @reactor_running
+      @connected || false
+    end
+
+    def unbind
+      Monga.logger.debug("Lost connection #{@host}:#{@port}")
+
+      @responses.each{ |k, cb| cb.call(Monga::Exceptions::LostConnection.new("Mongo has lost connection"))}
+      @connected = false
+      @primary = false
+      @pending_for_reconnect = false
+      set_deferred_status(nil)
+
+      if @reactor_running
+        EM.add_timer(0.1){ reconnect }
+      end
+    end
+
+    def close
+      Monga.logger.debug("EventMachine is stopped, closing connection")
+      @reactor_running = false
+    end
+
+    def master?
+      @primary || false
+    end
+
+    def is_master?(client)
+      db = client["admin"]
+      req = Monga::Requests::Query.new(db, "$cmd", query: {"isMaster" => 1}, limit: 1)
+      command = req.command
+      request_id = req.request_id
+      @responses[request_id] = proc do |data|
+        resp = req.parse_response(data)
+        if Exception === resp
+          @primary = false
+        else
+          @primary = resp.last.first["ismaster"]
+        end
+      end
+      send_data command
+    end
+  end
+end

data/lib/monga/connections/primary.rb

@@ -0,0 +1,46 @@
+module Monga::Connections
+  class Primary < Monga::Connections::EMConnection
+    def initialize(opts)
+      @client = opts.delete :client
+      @ser
+      super
+    end
+
+    def connection_completed
+      check_master do |master|
+        if master
+          super
+        else
+          reconnect
+        end
+      end
+    end
+
+    def check_master
+      db = @client["admin"]
+      request = Monga::Request.new(db, "$cmd", query: { "isMaster" => 1 })
+      request_id = request.request_id
+      @responses[request_id] = proc do |data|
+        request.parse_response(data)
+        if Exception === data
+          Monga.logger.debug("Error on connecting Primary to #{@host}:#{@port}, #{data.class}, #{data.message}")
+          @host, @port = @client.next_addr(@host, @port)
+          yield false
+        else
+          doc = data.last.first
+          if doc["ismaster"]
+            Monga.logger.debug("Primary has connected to #{@host}:#{@port}")
+            @client.inform(:primary, @host, @port)
+            yield true
+          else
+            Monga.logger.debug("#{@host}:#{@port} is not a primary")
+            @host, @port = @client.select_addr(doc)
+            yield false
+          end
+        end
+      end
+      command = request.command
+      send_data(command)
+    end
+  end
+end

data/lib/monga/connections/secondary.rb

@@ -0,0 +1,13 @@
+module Monga::Connections
+  class Secondary < Monga::Connections::EMConnection
+    def initialize(opts)
+      @client = opts.delete :client
+      super
+    end
+
+    def connection_completed
+      @client.inform(:secondary, @host, @port)
+      super()
+    end
+  end
+end