jor 0.1.1

data/lib/jor/doc.rb

@@ -0,0 +1,62 @@
+
+module JOR
+  class Doc
+
+    def self.paths(path,h)
+      if h.class==Hash
+        v = []
+        h.each do |k,val|
+          if JOR::Storage::SELECTORS_ALL.member?(k)
+            return [{"path_to" => path, "obj" => h, "class" => h.class, "selector" => true}]
+          else
+            raise InvalidFieldName.new(k) if ((k!="_id") && (k!="_created_at") && (k!="_updated_at")) && (k[0]=="_" || k[0]=="$")
+            v << paths("#{path}/#{k}",val)
+          end
+        end
+        return v.flatten
+      else
+        if h.class==Array
+          v = []
+          if h.size>0
+            h.each do |item|
+              v << paths("#{path}",item)
+           end
+          else
+            v << ["#{path}"]
+          end
+          return v.flatten
+        else
+          return [{"path_to" => path, "obj" => h, "class" => h.class}]
+        end
+      end
+    end
+
+    def self.difference(set, set_to_substract)
+      return set if set_to_substract.nil? || set_to_substract.size==0
+
+      to_exclude = []
+      set_to_substract.each do |item|
+        raise FieldIdCannotBeExcludedFromIndex.new unless item["path_to"].match(/\/_id/)==nil
+        to_exclude << Regexp.new("^#{item["path_to"]}")
+      end
+
+      res = []
+      set.each do |item|
+        not_found = true
+        to_exclude.each do |re|
+          not_found = not_found && re.match(item["path_to"])==nil
+        end
+        res << item if not_found
+      end
+      return res
+    end
+
+    def self.deep_merge(dest, source)
+      res = Hash.new
+      dest.merge(source) do |key, old_v, new_v|
+        res[key] = ((old_v.class == Hash) && (new_v.class == Hash))  ? deep_merge(old_v, new_v) : new_v
+      end
+    end
+
+  end
+end

data/lib/jor/errors.rb

@@ -0,0 +1,96 @@
+
+module JOR
+  class Error < RuntimeError
+  end
+
+  class NoResults < Error
+    def initialize(doc)
+      super %(no results found for "#{doc}")
+    end
+  end
+
+  class TypeNotSupported < Error
+    def initialize(class_name)
+      super %(Type #{class_name} not supported)
+    end
+  end
+
+  class InvalidFieldName < Error
+    def initialize(field)
+      super %(Invalid character in field name "#{field}". Cannot start with '_' or '$')
+    end
+  end
+
+  class InvalidDocumentId < Error
+    def initialize(id)
+      super %(The document "_id" must be a positive integer (>=0), #{id} is not valid)
+    end
+  end
+
+  class DocumentIdAlreadyExists < Error
+    def initialize(id, name)
+      super %(A document with _id #{id} already exists in collection "{name}")
+    end
+  end
+
+  class DocumentNeedsId < Error
+    def initialize(name)
+      super %(The collection #{name} is not auto-incremental. You must define the "_id" of the document")
+    end
+  end
+
+  class DocumentDoesNotNeedId < Error
+    def initialize(name)
+      super %(The collection #{name} is auto-incremental. You must not define the "_id" of the document")
+    end
+  end
+
+  class IncompatibleSelectors < Error
+    def initialize(str)
+      super %(Incompatible selectors in "#{str}". They must be grouped like this #{Storage::SELECTORS})
+    end
+  end
+
+  class NotInCollection < Error
+    def initialize
+      super %(The current collection is undefined)
+    end
+  end
+
+  class CollectionDoesNotExist < Error
+    def initialize(str)
+      super %(Collection "#{str}" does not exist)
+    end
+  end
+
+  class CollectionAlreadyExists < Error
+    def initialize(str)
+      super %(Collection "#{str}" already exists)
+    end
+  end
+
+  class CollectionNotValid < Error
+    def initialize(str)
+      super %(Collection "#{str}" is not a valid name, might be reserved)
+    end
+  end
+
+  class FieldIdCannotBeExcludedFromIndex < Error
+    def initialize
+      super %(Field _id cannot be excluded from the index)
+    end
+  end
+
+  class CouldNotFindPathToFromIndex < Error
+    def initialize(str)
+      super %(Could not find path_to from index #{str})
+    end
+  end
+
+  class CouldNotFindPathToFromIndex  < Error
+    def initialize(index, id)
+      super %(Unknown index #{index} in document #{id})
+    end
+  end
+
+end

data/lib/jor/server.rb

@@ -0,0 +1,52 @@
+
+module JOR
+  class Server
+
+    def initialize(redis = nil)
+      ## defaults to test db on redis
+      redis ||= Redis.new(:db => 9, :driver => :hiredis)
+      @jor = JOR::Storage.new(redis)
+    end
+
+    def call(env)
+      req = Rack::Request.new(env)
+      method = req.path
+
+      if env["REQUEST_METHOD"]!="PUT"
+        return [422, {"Content-Type" => "application/json"}, [{"error" => "only method accepted is PUT"}.to_json]]
+      end
+
+      clean_methods = method.gsub("/","").split(".")
+      clean_methods.map!(&:to_sym)
+
+      args = req.params["args"]
+      clean_args = []
+      body_str = req.body.read
+      clean_args = JSON::parse(body_str) unless body_str.nil? || body_str.empty?
+
+      begin
+        obj = @jor
+        res = nil
+
+        clean_methods.each_with_index do |meth, i|
+          if i==clean_methods.size()-1
+            res = obj.public_send(meth,*clean_args)
+          else
+            obj = obj.public_send(meth)
+          end
+        end
+
+        if res.class==Hash || res.class==Array
+          return [200, {"Content-Type" => "application/json"}, [res.to_json]]
+        else
+          return [200, {"Content-Type" => "application/json"}, [{"value" => res}.to_json]]
+        end
+      rescue Exception => e
+        return [422, {"Content-Type" => "application/json"}, [{"error" => e.message}.to_json]]
+      end
+    end
+  end
+end
+
+
+

data/lib/jor/storage.rb

@@ -0,0 +1,88 @@
+
+module JOR
+  class Storage
+
+    NAMESPACE = "jor"
+
+    SELECTORS = {
+      :compare => ["$gt","$gte","$lt","$lte"],
+      :sets => ["$in","$all","$not"]
+    }
+
+    SELECTORS_ALL = SELECTORS.keys.inject([]) { |sel, element| sel | SELECTORS[element] }
+
+    def initialize(redis_client = nil)
+      redis_client.nil? ? @redis = Redis.new() : @redis = redis_client
+    end
+
+    def redis
+      @redis
+    end
+
+    def collections
+      redis.smembers("#{Storage::NAMESPACE}/collections")
+    end
+
+    def create_collection(name, options = {:auto_increment => false})
+      options = {:auto_increment => false}.merge(options.inject({}){|memo,(k,v)| memo[k.to_sym] = v; memo})
+      raise CollectionNotValid.new(name) if self.respond_to?(name)
+      is_new = redis.sadd("#{Storage::NAMESPACE}/collections",name)
+      raise CollectionAlreadyExists.new(name) if (is_new==false or is_new==0)
+      redis.set("#{Storage::NAMESPACE}/collection/#{name}/auto-increment", options[:auto_increment])
+      name
+    end
+
+    def destroy_collection(name)
+      coll_to_be_removed = find_collection(name)
+      coll_to_be_removed.delete({})
+      redis.pipelined do
+        redis.srem("#{Storage::NAMESPACE}/collections",name)
+        redis.del("#{Storage::NAMESPACE}/collection/#{name}/auto-increment")
+      end
+      name
+    end
+
+    def destroy_all()
+      collections.each do |col|
+        destroy_collection(col)
+      end
+    end
+
+    def info
+      res = {}
+      ri = redis.info
+
+      res["used_memory_in_redis"] = ri["used_memory"].to_i
+      res["num_collections"] = collections.size
+
+      res["collections"] = {}
+      collections.each do |c|
+        coll = find_collection(c)
+        res["collections"][coll.name] = {}
+        res["collections"][coll.name]["num_documents"] = coll.count
+        res["collections"][coll.name]["auto_increment"] = coll.auto_increment?
+      end
+
+      res
+    end
+
+    protected
+
+    def method_missing(method)
+      find_collection(method)
+    end
+
+    def find_collection(method)
+      redis_auto_incr = redis.get("#{Storage::NAMESPACE}/collection/#{method}/auto-increment")
+      if (redis_auto_incr=="true")
+        auto_increment = true
+      elsif (redis_auto_incr=="false")
+        auto_increment = false
+      else
+        raise CollectionDoesNotExist.new(method.to_s)
+      end
+      Collection.new(self, method, auto_increment)
+    end
+
+  end
+end

data/lib/jor/version.rb

@@ -0,0 +1,3 @@
+module JOR
+  VERSION = '0.1.1'
+end

data/test/test_helper.rb

@@ -0,0 +1,11 @@
+require 'test/unit'
+require 'rack/test'
+require 'json'
+require 'hiredis'
+require 'redis'
+
+$:.unshift(File.expand_path(File.dirname(__FILE__) + '/../lib'))
+Dir[File.dirname(__FILE__) + '/test_helpers/**/*.rb'].each { |file| require file }
+Dir[File.dirname(__FILE__) + '/unit/test_case.rb'].each { |file| require file }
+
+require 'jor'

data/test/test_helpers/fixtures.rb

@@ -0,0 +1,71 @@
+module TestHelpers
+  module Fixtures
+
+    def create_sample_doc_cs(partial_doc = {})
+      doc = {
+        "_id" => 1,
+        "name" => {
+                   "first" => "John",
+                   "last" => "Backus"
+                 },
+        "birth" => Time.mktime("1924","12","03","05","00","00").to_i,
+        "death" => Time.mktime("2007","03","17","04","00","00").to_i,
+        "contribs" => [ "Fortran", "ALGOL", "Backus-Naur Form", "FP" ],
+        "awards" => [
+                    {
+                      "award" => "W.W. McDowellAward",
+                      "year" => 1967,
+                      "by" => "IEEE Computer Society"
+                    },
+                    {
+                      "award" => "National Medal of Science",
+                      "year" => 1975,
+                      "by" => "National Science Foundation"
+                    },
+                    {
+                      "award" => "Turing Award",
+                      "year" => 1977,
+                      "by" => "ACM"
+                    },
+                    {
+                      "award" => "Draper Prize",
+                      "year" => 1993,
+                      "by" => "National Academy of Engineering"
+                    }
+        ]
+      }
+      doc.merge(partial_doc)
+    end
+
+    def create_sample_doc_restaurant(partial_doc = {})
+      doc = {
+        "_id" => 1,
+        "name" => "restaurant",
+        "stars" => 3,
+        "cuisine" => ["asian", "japanese"],
+        "address" => {
+                       "address" => "Main St 100",
+                       "city"    => "Ann Arbor",
+                       "zipcode" => "08104"
+                     },
+        "description" => "very long description that we might not want to index",
+        "wines" => [
+                   {
+                     "name" => "wine1",
+                     "year" => 1998,
+                     "type" => ["garnatxa", "merlot"]
+                   },
+                   {
+                      "name" => "wine2",
+                      "year" => 2009,
+                      "type" => ["syrah", "merlot"]
+                   },
+        ]
+      }
+      doc.merge(partial_doc)
+    end
+
+  end
+end
+
+Test::Unit::TestCase.send(:include, TestHelpers::Fixtures)

data/test/unit/collection_test.rb

@@ -0,0 +1,884 @@
+require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
+
+class CollectionTest < JOR::Test::Unit::TestCase
+
+  def setup
+    super
+    @jor.create_collection("test")
+  end
+
+  def test_basic_insert_and_find_path
+
+    doc1 = create_sample_doc_restaurant({"_id" => 1})
+    @jor.test.insert(doc1)
+
+    doc2 = create_sample_doc_restaurant({"_id" => 2})
+    @jor.test.insert(doc2)
+
+    doc3 = create_sample_doc_restaurant({"_id" => 3})
+    @jor.test.insert(doc3)
+
+    assert_equal 3, @jor.test.count()
+    assert_equal 3, @jor.test.find({}).size
+
+    assert_equal doc1.to_json, @jor.test.find({"_id" => 1}).first.to_json
+    assert_equal doc2.to_json, @jor.test.find({"_id" => 2}).first.to_json
+    assert_equal doc3.to_json, @jor.test.find({"_id" => 3}).first.to_json
+  end
+
+  def test_bulk_insert
+
+    sample_docs = []
+    10.times do |i|
+      sample_docs << @jor.test.insert({"_id" => i, "name" => "foo_#{i}"})
+    end
+
+    assert_equal 10, @jor.test.count()
+
+    docs = @jor.test.find({})
+    10.times do |i|
+      assert_equal sample_docs[i].to_json, docs[i].to_json
+    end
+
+  end
+
+  def test_delete
+    sample_docs = []
+    10.times do |i|
+      sample_docs << @jor.test.insert({"_id" => i, "name" => "foo_#{i}", "foo" => "bar", "year" => 2000+i, "num" => 666 })
+    end
+
+    assert_equal 10, @jor.test.count()
+
+    assert_equal 0, @jor.test.delete({"_id" => 42})
+    assert_equal 10, @jor.test.count()
+
+    assert_equal 0, @jor.test.delete({"foo" => "not_bar"})
+    assert_equal 10, @jor.test.count()
+
+    assert_equal 1, @jor.test.delete({"_id" => 0})
+    assert_equal 9, @jor.test.count()
+
+    assert_equal 3, @jor.test.delete({"year" => { "$lt" => 2004 }})
+    assert_equal 6, @jor.test.count()
+
+    assert_equal ["4","5","6","7","8","9"].sort, @jor.redis.smembers("#{JOR::Storage::NAMESPACE}/test/idx/!/foo/String/bar").sort
+    assert_equal ["4","5","6","7","8","9"].sort, @jor.redis.smembers("#{JOR::Storage::NAMESPACE}/test/idx/!/num/Numeric/666").sort
+    assert_equal ["4","5","6","7","8","9"].sort, @jor.redis.zrange("#{JOR::Storage::NAMESPACE}/test/idx/!/num/Numeric",0,-1).sort
+
+    assert_equal 6, @jor.test.delete({"foo" => "bar"})
+    assert_equal 0, @jor.test.count()
+
+    assert_equal [], @jor.redis.smembers("#{JOR::Storage::NAMESPACE}/test/idx/!/num/String/bar")
+    assert_equal [], @jor.redis.smembers("#{JOR::Storage::NAMESPACE}/test/idx/!/num/Numeric/666")
+    assert_equal [].sort, @jor.redis.zrange("#{JOR::Storage::NAMESPACE}/test/idx/!/num/Numeric",0,-1).sort
+
+    assert_equal @jor.redis.keys("#{JOR::Storage::NAMESPACE}/test/idx/*"), []
+
+  end
+
+  def test_find_exact_string
+    sample_docs = []
+    10.times do |i|
+      sample_docs << @jor.test.insert({"_id" => i, "name" => "foo_#{i}"})
+    end
+
+    doc = @jor.test.find({"name" => "foo_5"}).first
+    assert_equal sample_docs[5].to_json, doc.to_json
+
+    doc = @jor.test.find({"name" => "foo_7"}).first
+    assert_equal sample_docs[7].to_json, doc.to_json
+  end
+
+  def test_find_empty
+    assert_equal [], @jor.test.find({})
+    assert_equal [], @jor.test.find({"year" => 200})
+    assert_equal [], @jor.test.find({"_id" => 200})
+  end
+
+  def test_find_by_comparison_selector
+
+    sample_docs = []
+    ## years from 2000 to 2009
+    10.times do |i|
+      sample_docs << @jor.test.insert({"_id" => i, "name" => "foo_#{i}", "year" => 2000+i})
+    end
+
+    doc = @jor.test.find({"year" => 2005}).first
+    assert_equal sample_docs[5].to_json, doc.to_json
+
+    doc = @jor.test.find({"year" => { "$lt" => 2005 }})
+    assert_equal 5, doc.size
+    assert_equal sample_docs[0].to_json, doc.first.to_json
+    assert_equal sample_docs[4].to_json, doc.last.to_json
+
+    doc = @jor.test.find({"year" => { "$lte" => 2005 }})
+    assert_equal 6, doc.size
+    assert_equal sample_docs[0].to_json, doc.first.to_json
+    assert_equal sample_docs[5].to_json, doc.last.to_json
+
+    doc = @jor.test.find({"year" => { "$gt" => 2007 }})
+    assert_equal 2, doc.size
+    assert_equal sample_docs[8].to_json, doc.first.to_json
+    assert_equal sample_docs[9].to_json, doc.last.to_json
+
+    doc = @jor.test.find({"year" => { "$gte" => 2007 }})
+    assert_equal 3, doc.size
+    assert_equal sample_docs[7].to_json, doc.first.to_json
+    assert_equal sample_docs[9].to_json, doc.last.to_json
+
+    doc = @jor.test.find({"year" => { "$gte" => 2003, "$lt" => 2005 }})
+    assert_equal 2, doc.size
+    assert_equal sample_docs[3].to_json, doc.first.to_json
+    assert_equal sample_docs[4].to_json, doc.last.to_json
+
+    doc = @jor.test.find({"year" => { "$gt" => 2003, "$lt" => 9999 }})
+    assert_equal 6, doc.size
+    assert_equal sample_docs[4].to_json, doc.first.to_json
+    assert_equal sample_docs[9].to_json, doc.last.to_json
+
+  end
+
+  def test_find_by_comparison_combined
+
+    sample_docs = []
+    ## years from 2000 to 2009
+    10.times do |i|
+      sample_docs << @jor.test.insert({"_id" => i, "year" => 2000+i, "desc" => "bar", "nested" => {"name" => "foo_#{i}", "quantity" => i.to_f}})
+    end
+
+    docs = @jor.test.find({"year" => { "$gt" => 2001, "$lt" => 2009 }, "nested" => {"quantity" => {"$lt" => 1.0}}})
+    assert_equal 0, docs.size
+    assert_equal [], docs
+
+    docs = @jor.test.find({"year" => { "$gt" => 2001, "$lt" => 2009 }, "nested" => {"quantity" => {"$lt" => 4.0}}})
+    assert_equal 2, docs.size
+    assert_equal sample_docs[2].to_json, docs.first.to_json
+    assert_equal sample_docs[3].to_json, docs.last.to_json
+
+    docs = @jor.test.find({"year" => { "$gt" => 2001, "$lt" => 2009 }, "nested" => {"name" => "foo_#{4}", "quantity" => {"$lte" => 4.0}}})
+    assert_equal 1, docs.size
+    assert_equal sample_docs[4].to_json, docs.first.to_json
+
+    docs = @jor.test.find({"year" => { "$gt" => 2001, "$lt" => 2009 }, "desc" => "bar", "nested" => {"name" => "foo_#{4}", "quantity" => {"$lte" => 4.0}}})
+    assert_equal 1, docs.size
+    assert_equal sample_docs[4].to_json, docs.first.to_json
+
+    docs = @jor.test.find({"year" => { "$gt" => 2001, "$lt" => 2009 }, "desc" => "NOT_bar", "nested" => {"name" => "foo_#{4}", "quantity" => {"$lte" => 4.0}}})
+    assert_equal 0, docs.size
+
+  end
+
+  def test_find_by_set_selector
+
+    sample_docs = []
+    ## years from 2000 to 2009
+    10.times do |i|
+      sample_docs << @jor.test.insert({"_id" => i, "name" => "foo_#{i}", "nested" => { "year" => 2000+i, "pair" => ((i%2)==0 ? "even" : "odd")} })
+    end
+
+    docs = @jor.test.find({"_id" => {"$in" => []}})
+    assert_equal 0, docs.size
+
+    docs = @jor.test.find({"_id" => {"$in" => [42]}})
+    assert_equal 0, docs.size
+
+    docs = @jor.test.find({"_id" => {"$in" => [8]}})
+    assert_equal 1, docs.size
+    assert_equal sample_docs[8].to_json, docs.first.to_json
+
+    docs = @jor.test.find({"_id" => {"$all" => [8]}})
+    assert_equal 1, docs.size
+    assert_equal sample_docs[8].to_json, docs.first.to_json
+
+    docs = @jor.test.find({"_id" => {"$in" => [1, 2, 3, 4, 42]}})
+    assert_equal 4, docs.size
+    assert_equal sample_docs[1].to_json, docs.first.to_json
+    assert_equal sample_docs[4].to_json, docs.last.to_json
+
+    docs = @jor.test.find({"_id" => {"$all" => [1, 2, 3, 4, 42]}})
+    assert_equal 0, docs.size
+
+    docs = @jor.test.find({"name" => {"$in" => ["foo_42"]}})
+    assert_equal 0, docs.size
+
+    docs = @jor.test.find({"name" => {"$all" => ["foo_42"]}})
+    assert_equal 0, docs.size
+
+    docs = @jor.test.find({"name" => {"$in" => []}})
+    assert_equal 0, docs.size
+
+    docs = @jor.test.find({"name" => {"$all" => []}})
+    assert_equal 0, docs.size
+
+    docs = @jor.test.find({"name" => {"$in" => ["foo_7", "foo_8", "foo_42"]}})
+    assert_equal 2, docs.size
+    assert_equal sample_docs[7].to_json, docs.first.to_json
+    assert_equal sample_docs[8].to_json, docs.last.to_json
+
+    docs = @jor.test.find({"nested" => {"pair" => { "$in" => ["even", "odd"]}}})
+    assert_equal 10, docs.size
+    assert_equal sample_docs[0].to_json, docs.first.to_json
+    assert_equal sample_docs[9].to_json, docs.last.to_json
+
+    docs = @jor.test.find({"nested" => {"pair" => { "$all" => ["even", "odd"]}}})
+    assert_equal 0, docs.size
+
+    docs = @jor.test.find({"nested" => {"pair" => { "$in" => ["even"]}}})
+    assert_equal 5, docs.size
+    assert_equal sample_docs[0].to_json, docs.first.to_json
+    assert_equal sample_docs[8].to_json, docs.last.to_json
+
+    docs = @jor.test.find({"nested" => {"pair" => { "$all" => ["even"]}}})
+    assert_equal 5, docs.size
+    assert_equal sample_docs[0].to_json, docs.first.to_json
+    assert_equal sample_docs[8].to_json, docs.last.to_json
+
+    docs = @jor.test.find({"nested" => {"pair" => { "$in" => ["even", "fake"]}}})
+    assert_equal 5, docs.size
+    assert_equal sample_docs[0].to_json, docs.first.to_json
+    assert_equal sample_docs[8].to_json, docs.last.to_json
+
+    docs = @jor.test.find({"nested" => {"pair" => { "$all" => ["even", "fake"]}}})
+    assert_equal 0, docs.size
+
+  end
+
+  def test_find_by_not_selector
+
+    sample_docs = []
+    ## years from 2000 to 2009
+    10.times do |i|
+      sample_docs << @jor.test.insert({"_id" => i, "name" => "foo_#{i}", "nested" => { "year" => 2000+i, "even" => ((i%2)==0 ? true : false)} })
+    end
+
+    docs = @jor.test.find({"_id" => {"$not" => 3}})
+    assert_equal 9, docs.size
+    docs.each do |doc|
+      assert_not_equal 3, doc["_id"]
+    end
+
+    docs = @jor.test.find({"_id" => {"$not" => [3]}})
+    assert_equal 9, docs.size
+    docs.each do |doc|
+      assert_not_equal 3, doc["_id"]
+    end
+
+    docs = @jor.test.find({"_id" => {"$not" => [1, 3, 4, 42]}})
+    assert_equal 7, docs.size
+    docs.each do |doc|
+      assert_not_equal 1, doc["_id"]
+      assert_not_equal 3, doc["_id"]
+      assert_not_equal 4, doc["_id"]
+      assert_not_equal 42, doc["_id"]
+    end
+
+    docs = @jor.test.find({"_id" => {"$not" => [3]}})
+    assert_equal 9, docs.size
+    docs.each do |doc|
+      assert_not_equal 3, doc["_id"]
+    end
+
+    docs = @jor.test.find({"name" => {"$not" => "foo_3"}})
+    assert_equal 9, docs.size
+    docs.each do |doc|
+      assert_not_equal 3, doc["_id"]
+    end
+
+    docs = @jor.test.find({"name" => {"$not" => ["foo_3"]}})
+    assert_equal 9, docs.size
+    docs.each do |doc|
+      assert_not_equal 3, doc["_id"]
+    end
+
+    docs = @jor.test.find({"name" => {"$not" => ["foo_1", "foo_3", "foo_4", "foo_42"]}})
+    assert_equal 7, docs.size
+    docs.each do |doc|
+      assert_not_equal 1, doc["_id"]
+      assert_not_equal 3, doc["_id"]
+      assert_not_equal 4, doc["_id"]
+      assert_not_equal 42, doc["_id"]
+    end
+
+    docs = @jor.test.find({"nested" => {"year" => {"$not" => 2004}}})
+    assert_equal 9, docs.size
+    docs.each do |doc|
+      assert_not_equal 4, doc["_id"]
+    end
+
+    docs = @jor.test.find({"nested" => {"year" => {"$not" => [2004, 2009]}}})
+    assert_equal 8, docs.size
+    docs.each do |doc|
+      assert_not_equal 4, doc["_id"]
+      assert_not_equal 9, doc["_id"]
+    end
+
+    docs = @jor.test.find({"nested" => {"even" => true}})
+    assert_equal 5, docs.size
+    docs.each do |doc|
+      assert_equal 0, doc["_id"]%2
+    end
+
+    docs = @jor.test.find({"nested" => {"even" => {"$not" => false}}})
+    assert_equal 5, docs.size
+    docs.each do |doc|
+      assert_equal 0, doc["_id"]%2
+    end
+
+
+
+
+  end
+
+  def test_playing_with_find_options
+
+    n = (JOR::Collection::DEFAULT_OPTIONS[:max_documents]+100)
+
+    n.times do |i|
+      doc = create_sample_doc_restaurant({"_id" => i})
+      @jor.test.insert(doc)
+    end
+
+    ## testing max_documents
+
+    docs = @jor.test.find({})
+
+    assert_equal JOR::Collection::DEFAULT_OPTIONS[:max_documents], docs.size
+    assert_equal 0, docs.first["_id"]
+    assert_equal JOR::Collection::DEFAULT_OPTIONS[:max_documents]-1, docs.last["_id"]
+
+    docs = @jor.test.find({},{:max_documents => 20})
+    assert_equal 20, docs.size
+
+    docs = @jor.test.find({},{:max_documents => -1})
+    assert_equal n, docs.size
+    assert_equal 0, docs.first["_id"]
+    assert_equal n-1, docs.last["_id"]
+
+    ## testing only_ids
+
+    docs = @jor.test.find({},{:only_ids => true, :max_documents => -1})
+    assert_equal n, docs.size
+    assert_equal 0, docs.first
+    assert_equal n-1, docs.last
+
+    docs = @jor.test.find({},{:only_ids => true})
+    assert_equal JOR::Collection::DEFAULT_OPTIONS[:max_documents], docs.size
+    assert_equal 0, docs.first
+    assert_equal JOR::Collection::DEFAULT_OPTIONS[:max_documents]-1, docs.last
+
+    ## testing reversed
+
+    docs = @jor.test.find({},{:only_ids => true, :max_documents => -1, :reversed => true})
+    assert_equal n, docs.size
+    assert_equal n-1, docs.first
+    assert_equal 0, docs.last
+
+    docs = @jor.test.find({},{:only_ids => true, :reversed => true})
+    assert_equal JOR::Collection::DEFAULT_OPTIONS[:max_documents], docs.size
+    assert_equal n-1, docs.first
+    assert_equal n-JOR::Collection::DEFAULT_OPTIONS[:max_documents], docs.last
+
+    ## encoded false
+
+    docs = @jor.test.find({},{:raw => true, :reversed => true})
+    assert_equal JOR::Collection::DEFAULT_OPTIONS[:max_documents], docs.size
+    assert_equal String, docs.first.class
+    assert_equal String, docs.last.class
+    assert_equal n-1, JSON::parse(docs.first)["_id"]
+    assert_equal n-JOR::Collection::DEFAULT_OPTIONS[:max_documents], JSON::parse(docs.last)["_id"]
+
+  end
+
+  def test_exclude_indexes
+
+    assert_raise JOR::FieldIdCannotBeExcludedFromIndex do
+      @jor.test.insert(create_sample_doc_restaurant({"_id" => 1}),
+        {:excluded_fields_to_index => {"_id" => true}})
+    end
+
+    @jor.test.insert(create_sample_doc_restaurant({"_id" => 1}),
+      {:excluded_fields_to_index => {"description" => true}})
+
+    @jor.test.insert(create_sample_doc_restaurant({"_id" => 42}),
+      {:excluded_fields_to_index => {}})
+
+    v = []
+    @jor.test.indexes(1).each do |ind|
+      v << ind["path"]
+    end
+    assert_equal false, v.include?("/description")
+
+    v = []
+    @jor.test.indexes(42).each do |ind|
+      v << ind["path"]
+    end
+    assert_equal true, v.include?("/description")
+
+    res = @jor.test.find({},{:reversed => true})
+    assert_equal 2, res.size
+    assert_equal "very long description that we might not want to index", res.first["description"]
+    assert_equal 42, res.first["_id"]
+    assert_equal "very long description that we might not want to index", res.last["description"]
+    assert_equal 1, res.last["_id"]
+
+    res = @jor.test.find({"description" => "very long description that we might not want to index"}, :reversed => true)
+    assert_equal 1, res.size
+    assert_equal 42, res.first["_id"]
+
+  end
+
+  def test_auto_increment_collections
+    @jor.create_collection("no_auto_increment")
+    @jor.create_collection("no_auto_increment2", :auto_increment => false)
+    @jor.create_collection("auto_increment",:auto_increment => true)
+
+    assert_equal false, @jor.no_auto_increment.auto_increment?
+    assert_equal false, @jor.no_auto_increment2.auto_increment?
+    assert_equal true, @jor.auto_increment.auto_increment?
+
+    10.times do |i|
+     @jor.auto_increment.insert({"foo" => "bar"})
+    end
+
+    assert_equal 10, @jor.auto_increment.count()
+
+    assert_raise JOR::DocumentDoesNotNeedId do
+     @jor.auto_increment.insert({"_id"=> 10, "foo" => "bar"})
+    end
+
+    assert_equal 10, @jor.auto_increment.count()
+    assert_equal 10, @jor.auto_increment.last_id()
+    assert_equal 0, @jor.no_auto_increment.last_id()
+  end
+
+  def test_last_id_for_collections
+    @jor.create_collection("no_auto_increment")
+    @jor.create_collection("auto_increment",:auto_increment => true)
+
+    assert_equal 0, @jor.no_auto_increment.last_id()
+    assert_equal 0, @jor.auto_increment.last_id()
+
+    10.times do |i|
+     @jor.auto_increment.insert({"foo" => "bar"})
+    end
+
+    assert_equal 10, @jor.auto_increment.last_id()
+    assert_not_nil @jor.auto_increment.find({"_id" => 10}).first
+
+    10.times do |i|
+     @jor.no_auto_increment.insert({"_id" => i+10, "foo" => "bar"})
+    end
+
+    assert_equal 19, @jor.no_auto_increment.last_id()
+    assert_not_nil @jor.no_auto_increment.find({"_id" => 10}).first
+  end
+
+  def test_ids_expections_for_collections
+    @jor.create_collection("no_auto_increment")
+    @jor.create_collection("auto_increment",:auto_increment => true)
+
+    assert_raise JOR::DocumentDoesNotNeedId do
+      @jor.auto_increment.insert({"_id"=> 10, "foo" => "bar"})
+    end
+
+    assert_raise JOR::DocumentNeedsId do
+      @jor.no_auto_increment.insert({"foo" => "bar"})
+    end
+
+    assert_raise JOR::InvalidDocumentId do
+      @jor.no_auto_increment.insert({"_id"=> "10", "foo" => "bar"})
+    end
+
+    assert_raise JOR::InvalidDocumentId do
+      @jor.no_auto_increment.insert({"_id"=> -1, "foo" => "bar"})
+    end
+  end
+
+  def test_ids_will_be_sorted
+   v = [1, 10, 100, 1000, 10000, 2, 20, 200, 2000, 20000]
+   v_sorted = v.sort
+   v_shuffled = v.shuffle
+
+   v_shuffled.each do |val|
+     @jor.test.insert({"_id" => val, "foo" => "bar"})
+   end
+
+   res = @jor.test.find({})
+   assert_equal v.size, res.size
+
+   res.each_with_index do |item, i|
+     assert_equal v_sorted[i], item["_id"]
+   end
+  end
+
+  def test_concurrent_inserts
+
+    threads = []
+    inserted_by_thread = []
+    5.times do |i|
+      inserted_by_thread[i] = []
+      threads << Thread.new {
+        id = i
+        1000.times do |j|
+          begin
+            doc = @jor.test.insert(create_sample_doc_restaurant({"_id" => j}))
+            inserted_by_thread[id] << j unless doc.nil?
+            sleep(0.1) if (j%200==0)
+          rescue Exception => e
+          end
+        end
+      }
+    end
+
+    threads.each do |t|
+      t.join()
+    end
+
+    res = @jor.test.find({},{:max_documents => -1})
+    assert_equal 1000, res.size
+
+    all = inserted_by_thread[0]
+    inserted_by_thread.each do |curr|
+      all =  all & curr
+    end
+    assert_equal 0, all.size
+
+    all = inserted_by_thread[0]
+    inserted_by_thread.each do |curr|
+      all =  all | curr
+    end
+    assert_equal 1000, all.size
+  end
+
+  def test_intervals
+    1000.times do |i|
+      @jor.test.insert(create_sample_doc_restaurant({"_id" => i, "at" => i}))
+    end
+
+    docs = @jor.test.find({})
+    assert_equal 1000, docs.size
+
+    docs = @jor.test.find({"at" => {"$lt" => 100}})
+    assert_equal 100, docs.size
+
+    docs = @jor.test.find({"at" => {"$lt" => 100, "$gte" => 50}})
+    assert_equal 50, docs.size
+
+    curr = 50
+    docs.each do |doc|
+      assert_equal curr, doc["_id"]
+      curr+=1
+    end
+
+    docs = @jor.test.find({"at" => {"$lt" => 100, "$gte" => 150}})
+    assert_equal 0, docs.size
+  end
+
+  def test_update
+    sample_docs = []
+    5.times do |i|
+      sample_docs << @jor.test.insert({"_id" => i, "name" => "foo_#{i}", "foo" => "bar", "year" => 2000+i })
+    end
+
+    @jor.test.update({"name" => "foo_4"}, {"name" => "foo_changed_4", "additional_field" => 3})
+
+    docs = @jor.test.find({"name" => "foo_4"})
+    assert_equal 0, docs.size()
+
+    [{"additional_field" => 3}, {"name" => "foo_changed_4"}, {"year" => 2004}].each do |search_doc|
+      docs = @jor.test.find(search_doc)
+      assert_equal 1, docs.size()
+      assert_equal "foo_changed_4", docs.first["name"]
+      assert_equal 3, docs.first["additional_field"]
+      assert_equal 2004, docs.first["year"]
+      assert_equal "bar", docs.first["foo"]
+    end
+
+  end
+
+  def test_update_with_id
+    sample_docs = []
+    5.times do |i|
+      sample_docs << @jor.test.insert({"_id" => i, "name" => "foo_#{i}", "foo" => "bar", "year" => 2000+i })
+    end
+
+    docs = @jor.test.update({"name" => "foo_4"}, {"name" => "foo_changed_4", "additional_field" => 3, "_id" => 666})
+    assert_equal 1, docs.size()
+    assert_equal "foo_changed_4", docs.first["name"]
+    assert_equal 4, docs.first["_id"]
+
+    docs = @jor.test.find({"_id" => 4})
+    assert_equal 1, docs.size()
+    assert_equal "foo_changed_4", docs.first["name"]
+    assert_equal 4, docs.first["_id"]
+  end
+
+  def test_update_with_excluded_fields
+    sample_docs = []
+    5.times do |i|
+      sample_docs << @jor.test.insert({"_id" => i, "name" => "foo_#{i}", "foo" => "bar", "year" => 2000+i })
+    end
+
+    docs = @jor.test.update({"foo" => "bar"}, {"foo" => "bar_changed", "description" => "long ass description"},
+      {:excluded_fields_to_index => {"description" => true}})
+
+    assert_equal 5, docs.size()
+    assert_equal "long ass description", docs.first["description"]
+    assert_equal "long ass description", docs.last["description"]
+
+    docs = @jor.test.find({"foo" => "bar_changed"})
+    assert_equal 5, docs.size()
+    assert_equal "long ass description", docs.first["description"]
+    assert_equal "long ass description", docs.last["description"]
+
+    docs = @jor.test.find({"description" => "long ass description"})
+    assert_equal 0, docs.size()
+
+    docs = @jor.test.update({"foo" => "bar_changed"}, {"description" => "long ass description"})
+    assert_equal 5, docs.size()
+
+    docs = @jor.test.find({"description" => "long ass description"})
+    assert_equal 5, docs.size()
+  end
+
+  def test_update_massive_update
+
+    sample_docs = []
+    5.times do |i|
+      sample_docs << @jor.test.insert({"_id" => i, "name" => "foo_#{i}", "foo" => "bar", "year" => 2000+i })
+    end
+
+    docs = @jor.test.find({"foo" => "bar"})
+    assert_equal 5, docs.size()
+
+    docs = @jor.test.find({"foo" => "bar_changed"})
+    assert_equal 0, docs.size()
+
+    docs = @jor.test.find({"bar" => "bar_added"})
+    assert_equal 0, docs.size()
+
+    @jor.test.update({"foo" => "bar"}, {"foo" => "bar_changed", "bar" => "bar_added"})
+
+    docs = @jor.test.find({"foo" => "bar"})
+    assert_equal 0, docs.size()
+
+    docs = @jor.test.find({"foo" => "bar_changed"})
+    assert_equal 5, docs.size()
+
+    docs = @jor.test.find({"bar" => "bar_added"})
+    assert_equal 5, docs.size()
+  end
+
+  def test_update_remove_field
+
+    sample_docs = []
+    5.times do |i|
+      sample_docs << @jor.test.insert({"_id" => i, "name" => "foo_#{i}", "foo" => "bar", "year" => 2000+i })
+    end
+
+    @jor.test.update({"foo" => "bar"}, {"foo" => nil, "xtra" => "large"})
+
+    docs = @jor.test.find({"foo" => "bar"})
+    assert_equal 0, docs.size()
+
+    docs = @jor.test.find({})
+    assert_equal 5, docs.size()
+
+    docs.each_with_index do |d, i|
+      assert_equal "foo_#{i}", d["name"]
+      assert_equal i, d["_id"]
+      assert_equal nil, d["foo"]
+      assert_equal "large", d["xtra"]
+    end
+
+    @jor.test.update({"xtra" => "large"}, {"foo" => "bar"})
+
+    docs = @jor.test.find({"foo" => "bar"})
+    assert_equal 5, docs.size()
+
+    docs.each_with_index do |d, i|
+      assert_equal "foo_#{i}", d["name"]
+      assert_equal i, d["_id"]
+      assert_equal "bar", d["foo"]
+      assert_equal "large", d["xtra"]
+    end
+
+  end
+
+  def test_update_remove_filed_nested
+
+    @jor.test.insert(create_sample_doc_restaurant({"_id" => 42}))
+    docs = @jor.test.find({"address" => {"zipcode" => "08104"}})
+    assert_equal 1, docs.size()
+
+    indexes_before = @jor.test.indexes(42)
+
+    @jor.test.update({"_id" => 42}, {"address" => {"zipcode" => nil}})
+    docs = @jor.test.find({"address" => {"zipcode" => "08104"}})
+    assert_equal 0, docs.size
+
+    indexes_after =  @jor.test.indexes(42)
+    ## -1  because we removed address/zipcode, but plus 1 because now _updated_at exists
+    assert_equal indexes_before.size, indexes_after.size
+    indexes_after.each do |item|
+      assert_equal false, ["/address/zipcode"].include?(item["path"])
+    end
+
+
+    docs = @jor.test.find({"_id" => 42})
+    assert_equal 1, docs.size
+    assert_equal "Ann Arbor", docs.first["address"]["city"]
+    assert_equal "Main St 100", docs.first["address"]["address"]
+
+    @jor.test.update({"_id" => 42}, {"address" => nil})
+    docs = @jor.test.find({"address" => {"zipcode" => "08104"}})
+    assert_equal 0, docs.size
+
+    indexes_after =  @jor.test.indexes(42)
+    assert_equal indexes_before.size - 2, indexes_after.size
+
+    indexes_after.each do |item|
+      assert_equal false, ["/address/zipcode", "/address/address", "/address/city"].include?(item["path"])
+    end
+
+  end
+
+  def test_update_arrays
+
+    sample_docs = []
+    5.times do |i|
+      sample_docs << @jor.test.insert({"_id" => i, "name" => "foo_#{i}", "foo" => [1,2,3,4,5], "year" => 2000+i })
+    end
+
+    @jor.test.update({"_id" => 4}, {"foo" => [6,7,8]})
+    docs = @jor.test.find({"_id" => 4})
+    assert_equal 1, docs.size()
+    assert_equal [6,7,8].sort, docs.first["foo"].sort
+
+    @jor.test.update({"_id" => 4}, {"foo" => {"bar" => [9, 10]}})
+    docs = @jor.test.find({"_id" => 4})
+    assert_equal 1, docs.size()
+    assert_equal [9, 10].sort, docs.first["foo"]["bar"].sort
+
+    @jor.test.update({"_id" => 4}, {"foo" => {"bar" => [11, 12]}})
+    docs = @jor.test.find({"_id" => 4})
+    assert_equal 1, docs.size()
+    assert_equal [11, 12].sort, docs.first["foo"]["bar"].sort
+
+    @jor.test.update({"_id" => 4}, {"foo" => nil})
+    docs = @jor.test.find({"_id" => 4})
+    assert_equal 1, docs.size()
+    assert_equal nil, docs.first["foo"]
+
+    assert_equal  0, ([{"path"=>"/name", "obj"=>"String", "value"=>"foo_4"},
+                    {"path"=>"/_id", "obj"=>"Numeric", "value"=>"4"},
+                    {"path"=>"/year", "obj"=>"Numeric", "value"=>"2004"}] - @jor.test.indexes(4)).size
+
+
+  end
+
+  def test_created_and_update_at
+    start_time = Time.now.to_f
+    sample_docs = []
+    5.times do |i|
+      sample_docs << @jor.test.insert({"_id" => i, "name" => "foo_#{i}", "foo" => [1,2,3,4,5], "year" => 2000+i })
+    end
+    end_time = Time.now.to_f
+
+    sample_docs.each do |d|
+      assert_equal true, d["_created_at"]>=start_time && d["_created_at"]<=end_time
+    end
+
+    docs = @jor.test.find({})
+    docs.each do |d|
+      assert_equal true, d["_created_at"]>=start_time && d["_created_at"]<=end_time
+      assert_equal nil, d["_updated_at"]
+    end
+
+    @jor.test.insert({"_id" => 42, "name" => "foo_#{42}", "foo" => [1,2,3,4,5], "year" => 2000+42, "_created_at" => start_time})
+    docs = @jor.test.find({"_id" => 42})
+    assert_equal start_time, docs.first["_created_at"]
+
+    start_time2 = Time.now.to_f
+    @jor.test.update({},{"updated" => "yeah"})
+    end_time2 = Time.now.to_f
+
+    docs = @jor.test.find({})
+    docs.each do |d|
+      assert_equal true, d["_created_at"]>=start_time && d["_created_at"]<=end_time
+      assert_equal true, d["_updated_at"]>=start_time2 && d["_updated_at"]<=end_time2
+    end
+
+    assert_equal 6, docs.size()
+
+    @jor.test.delete({"_created_at" => {"$lte" => start_time}})
+    docs = @jor.test.find({})
+    assert_equal 5, docs.size()
+  end
+
+  def test_boolean_fields
+
+    doc = @jor.test.insert({"_id" => 1, "foo" => "bar", "is_true" => true, "is_false" => false})
+    assert_equal true, doc["is_true"]
+    assert_equal false, doc["is_false"]
+
+    doc = @jor.test.insert({"_id" => 2, "foo" => "bar", "is_true" => true, "is_false" => false})
+    assert_equal true, doc["is_true"]
+    assert_equal false, doc["is_false"]
+
+    docs = @jor.test.find({})
+    assert_equal 2, docs.size
+    assert_equal true, docs.first["is_true"]
+    assert_equal false, docs.first["is_false"]
+    assert_equal true, docs.last["is_true"]
+    assert_equal false, docs.last["is_false"]
+
+    docs = @jor.test.find({"_id" => 2, "is_true" => true})
+    assert_equal 1, docs.size
+    assert_equal 2, docs.first["_id"]
+    assert_equal true, docs.first["is_true"]
+    assert_equal false, docs.first["is_false"]
+
+    docs = @jor.test.find({"_id" => 2, "foo" => "not_bar"})
+    assert_equal 0, docs.size
+
+    docs = @jor.test.find({"is_false" => false})
+    assert_equal 2, docs.size
+
+    docs = @jor.test.find({"_id" => 2, "is_true" => false})
+    assert_equal 0, docs.size
+
+    docs = @jor.test.find({"_id" => 2, "is_true" => "false"})
+    assert_equal 0, docs.size
+
+    docs = @jor.test.find({"_id" => 2, "is_true" => "true"})
+    assert_equal 0, docs.size
+
+    docs = @jor.test.find({"_id" => 2, "is_true" => nil})
+    assert_equal 0, docs.size
+
+    docs = @jor.test.update({"_id" => 2, "is_true" => true}, {"is_true" => false})
+    assert_equal 1, docs.size
+    assert_equal false, docs.first["is_true"]
+
+    docs = @jor.test.find({"_id" => 2})
+    assert_equal 1, docs.size
+    assert_equal 2, docs.first["_id"]
+    assert_equal false, docs.first["is_true"]
+    assert_equal false, docs.first["is_false"]
+
+    docs = @jor.test.find({"is_true" => {"$in" => [true, false]}})
+    assert_equal 2, docs.size
+    assert_equal true, docs.first["is_true"]
+    assert_equal false, docs.first["is_false"]
+    assert_equal 1, docs.first["_id"]
+    assert_equal false, docs.last["is_true"]
+    assert_equal false, docs.last["is_false"]
+    assert_equal 2, docs.last["_id"]
+
+    docs = @jor.test.find({"is_true" => {"$all" => [true, false]}})
+    assert_equal 0, docs.size
+  end
+
+end