monga 0.0.1

data/lib/monga/cursor.rb

@@ -0,0 +1,175 @@
+module Monga
+  class Cursor < EM::DefaultDeferrable
+    attr_reader :cursor_id
+
+    CURSORS = {}
+    CLOSED_CURSOR = 0
+    # Batch kill cursors marked to be killed each CLOSE_TIMEOUT seconds
+    CLOSE_TIMEOUT = 1
+
+    def initialize(db, collection_name, options = {}, flags = {})
+      @keep_alive = true if flags.delete :keep_alive
+
+      @db = db
+      @connection = @db.client.aquire_connection
+      @collection_name = collection_name
+      @options = options
+      @options.merge!(flags)
+
+      @fetched_docs = []
+      @count = 0
+      @total_count = 0
+      @limit = @options[:limit] ||= 0
+      @batch_size = @options[:batch_size]
+    end
+
+    def next_document
+      Monga::Response.surround do |resp|
+        if doc = @fetched_docs.shift
+          resp.succeed doc
+        else
+          req = next_batch
+          req.callback do |docs|
+            @fetched_docs = docs
+            if doc = @fetched_docs.shift
+              @count =+ 1
+              resp.succeed doc
+            end
+          end
+          req.errback{ |err| resp.fail err }
+        end
+      end
+    end
+
+    def each_doc(&blk)
+      if more?
+        req = next_batch
+        req.callback do |batch|
+          if batch.any?
+            batch.each do |doc|
+              @count += 1
+              blk.call(doc)
+            end
+            each_doc(&blk)
+          else
+            succeed
+          end
+        end
+        req.errback{ |err| fail err }
+      else
+        succeed
+      end
+      self
+    end
+
+    def kill
+      return unless @cursor_id > 0
+      self.class.kill_cursors(@connection, @cursor_id)
+      CURSORS.delete @cursor_id
+      @cursor_id = 0
+    end
+
+    def self.batch_kill(conn)
+      cursors = CURSORS.select{ |k,v| v }
+      cursor_ids = cursors.keys
+      if cursor_ids.any?
+        Monga.logger.debug("Following cursors are going to be deleted: #{cursor_ids}")
+        kill_cursors(conn, cursor_ids)
+        CURSORS.delete_if{|k,v| cursor_ids.include?(k) }
+      end
+    end
+
+    # Sometime in future all marked cursors will be killed in batch
+    def mark_to_kill
+      CURSORS[@cursor_id] = true if @cursor_id && alive?
+      @cursor_id = 0
+    end
+
+    # Cursor is alive and we need more minerals
+    def more?
+      alive? && !satisfied?
+    end
+
+    private
+
+    def get_more(batch_size)
+      Monga::Response.surround do |resp|
+        req = if @cursor_id
+          opts = { cursor_id: @cursor_id, batch_size: batch_size }
+          Monga::Requests::GetMore.new(@db, @collection_name, opts).callback_perform
+        else
+          Monga::Requests::Query.new(@db, @collection_name, @options).callback_perform
+        end
+        req.callback do |data|
+          @cursor_id = data[5]
+          fetched_docs = data.last
+          @total_count += fetched_docs.count
+          mark_to_kill unless cursor_more?
+
+          resp.succeed fetched_docs
+        end
+        req.errback do |err|
+          mark_to_kill
+          resp.fail err
+        end
+      end
+    end
+
+    def next_batch
+      Monga::Response.surround do |resp|
+        if more?
+          batch_size = get_batch_size
+          req = get_more(batch_size)
+          req.callback{ |res| resp.succeed res }
+          req.errback{ |err| resp.fail err }
+        else
+          mark_to_kill
+          if !alive?
+            resp.fail Monga::Exceptions::CursorIsClosed.new("Cursor is already closed. Check `cursor.more?` before calling cursor")
+          elsif satisfied?
+            resp.fail Monga::Exceptions::CursorLimit.new("You've already fetched #{@limit} docs you asked. Check `cursor.more?` before calling cursor")
+          end
+        end
+      end
+    end
+
+    def cursor_more?
+      alive? && !cursor_satisfied?
+    end
+
+    # If cursor_id is not setted, or if isn't CLOSED_CURSOR - cursor is alive
+    def alive?
+      @cursor_id != CLOSED_CURSOR
+    end
+
+    # If global limit is setted 
+    # we will be satisfied when we will get limit amount of documents.
+    # Otherwise we are not satisfied untill crsor is alive
+    def satisfied?
+      @limit > 0 && @count >= @limit
+    end
+
+    def cursor_satisfied?
+      @limit > 0 && @total_count >= @limit
+    end
+
+    # How many docs should be returned
+    def rest
+      @limit - @count if @limit > 0
+    end
+
+    # Cursor will get exact amount of docs as user passed with `limit` opr
+    def get_batch_size
+      if @limit > 0 && @batch_size
+        rest < @batch_size ? rest : @batch_size
+      else @batch_size
+        @batch_size
+      end
+    end
+
+    def self.kill_cursors(connection, cursor_ids)
+      Monga::Requests::KillCursors.new(connection, cursor_ids: [*cursor_ids]).perform
+    end
+
+  end
+end

data/lib/monga/database.rb

@@ -0,0 +1,97 @@
+module Monga
+  class Database
+    attr_reader :client, :name
+
+    def initialize(client, name)
+      @client = client
+      @name = name
+    end
+
+    def [](collection_name)
+      Monga::Collection.new(self, collection_name)
+    end
+
+    def cmd(cmd, opts={})
+      options = {}
+      options[:query] = cmd
+      options.merge! opts
+      Monga::Miner.new(self, "$cmd", options).limit(1)
+    end
+
+    def eval(js)
+      with_response do
+        cmd(eval: js)
+      end
+    end
+
+    # Be carefull with using get_last_error with connection pool.
+    # In most cases you need to use #safe methods 
+    # and don't access to #get_last_error directky
+    def get_last_error
+      with_response do
+        cmd(getLastError: 1)
+      end
+    end
+
+    def drop_collection(collection_name)
+      with_response do
+        cmd(drop: collection_name)
+      end
+    end
+
+    def create_collection(collection_name, opts = {})
+      with_response do
+        cmd({create: collection_name}.merge(opts))
+      end
+    end
+
+    def count(collection_name)
+      Monga::Response.surround do |resp|
+        req = with_response do
+          cmd(count: collection_name)
+        end
+        req.callback do |data|
+          cnt = data.first["n"].to_i
+          resp.succeed cnt
+        end
+        req.errback{ |err| resp.fail err }
+      end
+    end
+
+    def drop_indexes(collection_name, indexes)
+      with_response do
+        cmd(dropIndexes: collection_name, index: indexes)
+      end
+    end
+
+    # Just helper
+    def list_collections
+      Monga::Response.surround do |resp|
+        req = eval("db.getCollectionNames()")
+        req.callback do |data|
+          resp.succeed(data.first["retval"])
+        end
+        req.errback do |err|
+          resp.fail(err)
+        end
+      end
+    end
+
+    private
+
+    def with_response
+      Monga::Response.surround do |resp|
+        req = yield(resp)
+        req.callback do |data|
+          if data.any?
+            resp.succeed(data)
+          else
+            exception = Monga::Exceptions::QueryFailure.new("Nothing was returned for your query: #{req.options[:query]}")
+            resp.fail(exception)
+          end
+        end
+        req.errback{ |err| resp.fail err }
+      end
+    end
+  end
+end

data/lib/monga/exceptions.rb

@@ -0,0 +1,9 @@
+module Monga::Exceptions
+  class LostConnection < StandardError; end
+  class CursorNotFound < StandardError; end
+  class CursorIsClosed < StandardError; end
+  class CursorLimit < StandardError; end
+  class QueryFailure < StandardError; end
+  class UndefinedIndexVersion < StandardError; end
+  class NoAvailableServers < StandardError; end
+end

data/lib/monga/miner.rb

@@ -0,0 +1,72 @@
+# Miner is a "proxy" object to Cursor.
+# It dinamically stores Cursor options and at any moment can return cursor.
+# Also it hides Deferrable that could return all objects that cursor do.
+module Monga
+  class Miner < EM::DefaultDeferrable
+    attr_reader :options
+    
+    def initialize(db, collection_name, options={})
+      @db = db
+      @collection_name = collection_name
+      @options = options
+
+      # Defaults
+      @options[:query] ||= {}
+      @options[:limit] ||= 0
+      @options[:skip] ||= 0
+    end
+
+    def cursor(flags = {})
+      @cursor = Monga::Cursor.new(@db, @collection_name, @options, flags)
+    end
+
+    def explain
+      @options[:explain] = true
+    end
+
+    def hint
+      @options[:hint] = true
+    end
+
+    def sort(val)
+      @options[:sort] = val
+    end
+
+    def limit(count)
+      @options[:limit] = count and self
+    end
+
+    def skip(count)
+      @options[:skip] = count and self
+    end
+
+    def batch_size(count)
+      @options[:batch_size] = count and self
+    end
+
+    # Lazy operation execution
+    [:callback, :errback, :timeout].each do |meth|
+      class_eval <<-EOS
+        def #{meth}(*args)
+          mine! && @deferred = true unless @deferred
+          super
+        end
+      EOS
+    end
+
+    private
+
+    def mine!
+      docs = []
+      itrator = cursor.each_doc do |doc|
+        docs << doc
+      end
+      itrator.callback do |resp|
+        succeed docs
+      end
+      itrator.errback do |err|
+        fail err
+      end
+    end
+  end
+end

data/lib/monga/request.rb

@@ -0,0 +1,122 @@
+module Monga
+  class Request
+    attr_reader :request_id
+
+    OP_CODES = {
+      reply:           1,
+      msg:          1000,
+      update:       2001,
+      insert:       2002,
+      reserved:     2003,
+      query:        2004,
+      get_more:     2005,
+      delete:       2006,
+      kill_cursors: 2007,
+    }
+
+    def initialize(db, collection_name, options = {})
+      @db = db
+      @collection_name = collection_name
+      @options = options
+      @request_id = self.class.request_id
+      @connection = @db.client.aquire_connection
+    end
+
+    def command
+      header.append!(body)
+    end
+
+    def header
+      headers = BSON::ByteBuffer.new
+      headers.put_int(command_length)
+      headers.put_int(@request_id)
+      headers.put_int(0)
+      headers.put_int(op_code)
+      headers
+    end
+
+    # Fire and Forget
+    def perform
+      @connection.send_command(command)
+      @request_id
+    end
+
+    # Fire and wait
+    def callback_perform
+      Monga::Response.surround do |response|
+        @connection.send_command(command, @request_id) do |data|
+          resp = parse_response(data)
+          Exception === resp ? response.fail(resp) : response.succeed(resp)
+        end
+      end
+    end
+
+    def parse_response(data)
+      if Exception === data
+        data
+      else
+        flags = data[4]
+        number = data[7]
+        docs = unpack_docs(data.last, number)
+        data[-1] = docs
+        if flags & 2**0 > 0
+          Monga::Exceptions::CursorNotFound.new(docs.first)
+        elsif flags & 2**1 > 0
+          Monga::Exceptions::QueryFailure.new(docs.first)
+        elsif docs.first && (docs.first["err"] || docs.first["errmsg"])
+          Monga::Exceptions::QueryFailure.new(docs.first)
+        else
+          data
+        end
+      end
+    end
+
+    private
+
+    def unpack_docs(data, number)
+      number.times.map do
+        size = data.slice(0, 4).unpack("L").first
+        d = data.slice!(0, size)
+        BSON.deserialize(d)
+      end
+    end
+
+    def flags
+      flags = 0
+      self.class::FLAGS.each do |k, byte|
+        flags = flags | 1 << byte if @options[k]
+      end
+      flags
+    end
+
+    def full_name
+      [@db.name, @collection_name] * "."
+    end
+
+    def op_code
+      OP_CODES[self.class.op_name]
+    end
+
+    def command_length
+      HEADER_SIZE + body.size
+    end
+
+    def self.request_id
+      @request_id ||= 0
+      @request_id += 1
+      @request_id >= 2**32 ? @request_id = 1 : @request_id
+    end
+
+    def self.op_name(op = nil)
+      op ? @op_name = op : @op_name
+    end
+  end
+end
+
+
+require File.expand_path("../requests/query", __FILE__)
+require File.expand_path("../requests/insert", __FILE__)
+require File.expand_path("../requests/delete", __FILE__)
+require File.expand_path("../requests/update", __FILE__)
+require File.expand_path("../requests/get_more", __FILE__)
+require File.expand_path("../requests/kill_cursors", __FILE__)

data/lib/monga/requests/delete.rb

@@ -0,0 +1,23 @@
+module Monga::Requests
+  class Delete < Monga::Request
+    op_name :delete
+
+    FLAGS = { 
+      single_remove: 0,
+    }
+
+    def body
+      @body ||= begin
+        query = @options[:query]
+
+        b = BSON::ByteBuffer.new
+        b.put_int(0)
+        BSON::BSON_RUBY.serialize_cstr(b, full_name)
+        b.put_int(flags)
+        b.append!(BSON::BSON_C.serialize(query).to_s)
+        b
+      end
+    end
+
+  end
+end

data/lib/monga/requests/get_more.rb

@@ -0,0 +1,19 @@
+module Monga::Requests
+  class GetMore < Monga::Request
+    op_name :get_more
+    
+    def body
+      @body ||= begin
+        batch_size = @options[:batch_size] || 0
+        cursor_id = @options[:cursor_id]
+
+        b = BSON::ByteBuffer.new
+        b.put_int(0)
+        BSON::BSON_RUBY.serialize_cstr(b, full_name)
+        b.put_int(batch_size)
+        b.put_long(cursor_id)
+        b
+      end
+    end
+  end
+end

data/lib/monga/requests/insert.rb

@@ -0,0 +1,29 @@
+module Monga::Requests
+  class Insert < Monga::Request
+    op_name :insert
+
+    FLAGS = { 
+      continue_on_error: 0,
+    }
+
+    def body
+      @body ||= begin
+        documents = @options[:documents]
+
+        b = BSON::ByteBuffer.new
+        b.put_int(flags)
+        BSON::BSON_RUBY.serialize_cstr(b, full_name)
+        case documents
+        when Array
+          documents.each do |doc|
+            b.append!(BSON::BSON_C.serialize(doc).to_s)
+          end
+        when Hash
+          b.append!(BSON::BSON_C.serialize(documents).to_s)
+        end
+        b
+      end
+    end
+
+  end
+end

data/lib/monga/requests/kill_cursors.rb

@@ -0,0 +1,25 @@
+module Monga::Requests
+  class KillCursors < Monga::Request
+    op_name :kill_cursors
+
+    def initialize(connection, options = {})
+      @options = options
+      @request_id = self.class.request_id
+      @connection = connection
+    end
+    
+    def body
+      @body ||= begin
+        cursor_ids = @options[:cursor_ids]
+
+        b = BSON::ByteBuffer.new
+        b.put_int(0)
+        b.put_int(cursor_ids.size)
+        cursor_ids.each do |cursor_id|
+          b.put_long(cursor_id)
+        end
+        b
+      end
+    end
+  end
+end

data/lib/monga/requests/query.rb

@@ -0,0 +1,49 @@
+module Monga::Requests
+  class Query < Monga::Request
+    op_name :query
+
+    FLAGS = {
+      tailable_cursor: 1,
+      slave_ok: 2,
+      no_cursor_timeout: 4,
+      await_data: 5,
+      exhaust: 6,
+      partial: 7,
+    }
+
+    def body
+      @body ||= begin
+        skip = @options[:skip] || 0
+        limit = get_limit
+        fields = @options[:fields] || {}
+
+        query = {}
+        query["$query"] = @options[:query] || {}
+        query["$hint"] = @options[:hint] if @options[:hint]
+        query["$orderby"] = @options[:sort] if @options[:sort]
+        query["$explain"] = @options[:explain] if @options[:explain]
+
+        b = BSON::ByteBuffer.new
+        b.put_int(flags)
+        BSON::BSON_RUBY.serialize_cstr(b, full_name)
+        b.put_int(skip)
+        b.put_int(limit)
+        b.append!(BSON::BSON_C.serialize(query).to_s)
+        b.append!(BSON::BSON_C.serialize(fields).to_s) if fields.any?
+        b
+      end
+    end
+
+    private
+
+    def get_limit
+      if @options[:batch_size]
+        @options[:batch_size]
+      elsif @options[:limit]
+        -@options[:limit]
+      else
+        0
+      end
+    end
+  end
+end

data/lib/monga/requests/update.rb

@@ -0,0 +1,25 @@
+module Monga::Requests
+  class Update < Monga::Request
+    op_name :update
+
+    FLAGS = { 
+      upsert: 0,
+      multi_update: 1,
+    }
+
+    def body
+      @body ||= begin
+        query = @options[:query]
+        update = @options[:update]
+
+        b = BSON::ByteBuffer.new
+        b.put_int(0)
+        BSON::BSON_RUBY.serialize_cstr(b, full_name)
+        b.put_int(flags)
+        b.append!(BSON::BSON_C.serialize(query).to_s)
+        b.append!(BSON::BSON_C.serialize(update).to_s)
+        b
+      end
+    end
+  end
+end

data/lib/monga/response.rb

@@ -0,0 +1,11 @@
+module Monga
+  class Response
+    include EM::Deferrable
+
+    def self.surround
+      resp = new
+      yield(resp)
+      resp
+    end
+  end
+end

data/lib/monga.rb

@@ -0,0 +1,30 @@
+require "eventmachine"
+require "bson"
+require "logger"
+
+module Monga
+  DEFAULT_HOST = "127.0.0.1"
+  DEFAULT_PORT = 27017
+  HEADER_SIZE  = 16
+  
+  extend self
+
+  def logger
+    @logger ||= begin
+      l = Logger.new(STDOUT)
+      l.level = Logger::DEBUG
+      l
+    end
+  end
+end
+
+require File.expand_path("../monga/connection", __FILE__)
+require File.expand_path("../monga/connection_pool", __FILE__)
+require File.expand_path("../monga/client", __FILE__)
+require File.expand_path("../monga/database", __FILE__)
+require File.expand_path("../monga/collection", __FILE__)
+require File.expand_path("../monga/miner", __FILE__)
+require File.expand_path("../monga/cursor", __FILE__)
+require File.expand_path("../monga/exceptions", __FILE__)
+require File.expand_path("../monga/response", __FILE__)
+require File.expand_path("../monga/request", __FILE__)