jor 0.1.1

data/.gitignore

@@ -0,0 +1,3 @@
+/*.gem
+/tmp/*
+Gemfile.lock

data/Gemfile

@@ -0,0 +1,2 @@
+source :rubygems
+gemspec

data/LICENCE

@@ -0,0 +1,22 @@
+The MIT License
+
+Copyright (c) 2013 3scale networks S.L.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+

data/README.md

@@ -0,0 +1,319 @@
+
+# JOR
+
+## Description
+
+**JOR** is the acronym for **J**SON **o**ver **R**edis.
+
+This project provides persistent storage for objects that can be serialized in JSON.
+
+JOR uses Redis as the backend data-store, furthermore, it provides a MongoDB-like query language for fast retrieval that is heavily inspired by the API of MongoDB.
+
+Why not using MongoDB right away? There is no a good reason :) JOR is leaner but only offers a fraction of the functionality. I would not consider JOR for the backend
+of any large system but go straight to MongoDB. However, if you project is small enough, or it has to be embedded, or if you are already using Redis for some other task you might
+find it interesting.
+
+JOR aims to only offer:
+
+* CRUD for objects serialized as JSON documents, and
+* a JSON-based query language to find the documents matching the constrains of the query document
+
+For instance, the document (in this case a ruby Hash)
+
+    doc = {
+       "_id" => 42,
+       "name" => "bon menjar",
+       "stars" => 3,
+       "cuisine" => ["asian", "japanese"],
+       "address" => {
+                      "address" => "Main St 100",
+                      "city"    => "Ann Arbor",
+                      "zipcode" => "08104"
+                    },
+       "description" => "very long description that we might not want to index",
+       "wines" => [
+                  {
+                    "name" => "wine1",
+                    "year" => 1998,
+                    "type" => ["garnatxa", "merlot"]
+                  },
+                  {
+                     "name" => "wine2",
+                     "year" => 2009,
+                     "type" => ["syrah", "merlot"]
+                  }
+       ]
+     }
+
+
+can be inserted like this:
+
+    jor.create_collection("restaurants")
+    jor.restaurants.insert(doc)
+
+and retrieved by using a query (that is also a document). For instance:
+
+    jor.restaurants.find({})
+
+will return all documents in the `restaurants` collection.
+
+    jor.restaurants.find({"_id" => 42})
+
+will return all documents whose `_id` is 42. The query document can be arbitrarily complex:
+
+    jor.restaurants.find({
+      "stars" => {"$gte" => 3},
+      "wines" => {"type" =>
+        {"$in" => ["garnatxa", "syrah"]}}
+    })
+
+the `find` will return restaurants with 3 or more stars that also have wines of type
+garnatxa or syrah.
+
+
+## Getting Started
+
+### Installation
+
+From rubygems
+
+    gem install jor
+
+From source
+
+    git clone https://github.com/solso/jor.git
+    build jor.gemspec
+    gem install
+
+
+### Initialize
+
+You can pass the your own redis handler on instantiation…
+
+    require 'jor'
+
+    redis = Redis.new(:driver => :hiredis)
+    jor = JOR::Storage.new(redis)
+
+if you don't, JOR will create a redis connection against redis' default (localhost, port 6379).
+
+We advise using `hiredis` to improve performance.
+
+JOR is not thread-safe at the current version.
+
+## How to
+
+### Collections
+
+JOR allows to have multiple **collections**. A collections is a container of documents that are mean to be group together. The operations `insert`, `delete` and `find` are scoped by a collection.
+
+To create a collection:
+
+    jor.create_collection("restaurants")
+
+Number of documents in collection:
+
+    jor.restaurant.count()
+
+Id of the last document inserted in the collection, zero if empty
+
+    jor.restaurant.last_id()
+
+Delete a collection with all its documents:
+
+    jor.destroy_collection("restaurants")
+
+Delete all collections:
+
+    jor.destroy_all()
+
+Collections can be created to have **auto_incremental** ids
+
+    jor.create_collection("events", :auto_increment => true)
+
+A collection is either auto_incremental or not, cannot be both types at the same time. The default type is not auto-incremental.
+
+Auto-incremental collections expect documents without the field `_id`, which will be assigned automatically upon insertion.
+
+### Insert
+
+To insert documents to a collection just do `insert`. The parameters can be either a `Hash` (will be stored as JSON encoded string), or an `Array` of `Hash` objects
+
+    jor.restaurant.insert(doc)
+    jor.restaurant.insert([doc1, doc2, …, docN])
+
+There is marginal benefits to use bulk insertion, it's mostly for convenience.
+
+Every document stored in JOR has an field called **´_id´** that is unique within a collection. Trying to insert a document with an `_id` that already exists will raise an exception.
+
+The `_id` must be a natural number (>=0), remember that you only need to define the field `_id` when dealing with collections that are not auto-incremental (the default case).
+
+By the way, field names cannot start with `'` or with `$`. These characters are reserved.
+
+There are two other special fields:
+
+* `_created_at`: it is set when inserting a document, and should not changed ever again. The field is indexed by default.
+* `_updated_at`: it is set every time the document is updated. Also indexed by default.
+
+Both times are unix epoch time in milli-seconds (the decimal part).
+
+#### Options:
+
+`:exclude_fields_to_index`
+
+If you know that you will never do a search for a field you might want to exclude it from the indexes. By default all fields are indexes. Adding fields to be excluded improves the efficiency of the insert.
+
+For instance, if you want to exclude the field `description` from the index:
+
+    jor.restaurant.insert(doc,
+        {:excluded_fields_to_index => {"description" => true}})
+
+Excluding fields is something to consider because the performance of the insert in linear with the number of fields of the document O(#fields). An excluded fields will not affect the content of the document, it will just make it not "findable".
+
+We advise to exclude any fields that is a string that does not serve as a symbol or tag since strings fields can only be found by a exact match.
+
+You can also exclude fields that are objects, for instance, if you do not want to index the types of wines:
+
+    jor.restaurant.insert(doc,
+        {:excluded_fields_to_index => {"wines" => {"type" => true}}})
+
+Exclusion is per document based, it will only affect the document being inserted.
+
+The field `_id` cannot be excluded.
+
+Note that if you exclude a field from the index you will not be able to use that field on `find` operations. Search is only done over indexed fields. Unless explicitly stated all fields of the documents are indexed.
+
+
+### Find
+
+To retrieve the stored documents you only need to define a query document (also a Hash). The interface is inspired on the MongoDB query language, so if you are familiar with it will be straight forward.
+
+    jor.restaurant.find({})
+
+will find all restaurants in the collection. The query document is all `{}`.
+
+The query document `{"_id" => 42}` will only return one (or zero) restaurant documents. The one whose field `_id` has value 42.
+
+The query document is a subset of the original stored document. For the fields defined, it will match the value. For those who are not defined, it will act as a wildcard.
+
+Some `operators` are also available:
+
+* For comparisons:
+  * **$gt**: greater than (>)
+  * **$gte**: greater than or equal (>=)
+  * **$lt**: lower than (<)
+  * **$lte**: lower than or equal (<=)
+
+* For sets:
+  * **$in**: the value must be in the defined set
+  * **$all**: all values must be in the defined set
+  * **$not**: the value must not be in the defined set
+
+The syntax to use the `operators` also follows a hash
+
+    jor.restaurants.find({
+      "stars" => {"$gte" => 3},
+      "wines" => {
+        "year" => 2008,
+        "type" => {
+          "$all" => ["garnatxa", "syrah"]
+        }
+      }
+    })
+
+The query document will return all documents that match all 3 conditions:
+
+The field `start` must be greater or equal than 3, they have at least one `wine` that is from `year` 2008 and the `type` of wine contains both garnatxa and syrah (turns out that wines can be mixed)
+
+The following `find` returns all documents whose `_id` is on the list
+
+    jor.restaurants.find({"_id" => {"$in" => [1, 3, 5, 7, 11]}})
+
+The result of the `find` in an Array of the documents (as Hash objects). The documents are returned by ascending `_id`.
+
+The following `find` returns all documents except those whose `name` is "el bulli" or "can roca"
+
+    jor.restaurants.find({"name" => {"$not" => ["el bulli", "can roca"]}})
+
+
+
+#### Options:
+
+`find` accepts the following options that you can override:
+
+* `:max_documents`, the maximum number of documents to be returned, by default 1000.
+* `:only_ids`, return only the ids instead of the Hash, by default false. This is useful for joins.
+* `:raw`, returns only the document as JSON encoded strings, you save JOR to do the final parsing of the JSON encoded string. By default false.
+* `:reversed`, returns the documents sorted by descendant `_id`. Default if false.
+
+
+### Delete
+
+Deleting a document is basically like doing a `find` with the exception that all documents that meet the query document will be deleted.
+
+    jor.restaurants.find({"_id" => 42})
+
+Deletes the document with `_id` 42 (only one document by definition).
+
+    jor.restaurants.delete({"address" => {"zipcode" => "08104"}})
+
+Deletes any document that the `zipcode` on its `address` is "08104".
+
+
+### Update
+
+Updating a document is also doing a `find` and doing a deep merge of the documents found and the source doc. For instance,
+
+    jor.restaurants.update({"_id" => 42}, {"address" => {"zipcode" => "08105"}})
+
+Updates (or add if did not exist) the `address` => `zipcode` of the document with `_id` 42.
+
+    jor.restaurants.update({"address" => {"zipcode" => "08105"}} , {"address" => {"zipcode" => "08106"}})
+
+Updates all documents with `zipcode` "08105" to "08106". Updates are __not__ limited to a single document. The update applies
+to all the documents that match the first argument of the `update` operation.
+
+Indexes are managed in the same way than an `insert` operations, so that you can use `:exclude_fields_to_index` as options.
+
+If the update is a removal of a field, you must do it like this:
+
+    jor.restaurants.update({"_id" => 42}, {"address" => nil)
+
+Note that this will remove all the fields that hang from `address`, whether it is a value, array or a hash.
+
+### Misc Operations
+
+You can find which document fields are indexed by doing,
+
+    jor.restaurants.indexes(doc_id)
+
+This operation will return all fields that are indexed, but not all the indexes there are. Numeric fields, for instance, have two indexes.
+
+Also, you can get sysadmin related info,
+
+    jor.info()
+
+
+## Benchmarks
+
+The thing is quite fast (thanks to Redis).
+
+With a commodity laptop (macbook air) we can get between 300~400 documents inserts per second for the restaurant example used in this README.
+
+The complexity of an `insert` and `find` operations depend on the number of fields of the document and the query document respectively. For the case of the restaurant document there are 16 fields.
+
+Real benchmarks are more than welcomed.
+
+## To Do
+
+* normalize indexed strings (downcase, trimmed, something else) so that at least the == on a string is case insensitive. Tokenizing a string is easy to do, but can affect performance O(#fields + #words_on_string_fields). Perhaps as an option.
+
+## Contribute
+
+Fork the [project](http://github.com/solso/jor/) and send pull requests.
+
+
+
+
+
+

data/Rakefile

@@ -0,0 +1,10 @@
+require 'rake/testtask'
+require 'bundler/gem_tasks'
+
+desc 'Run all unit tests'
+
+Rake::TestTask.new(:test) do |task|
+  task.test_files = FileList['test/unit/**/*_test.rb']
+  task.verbose = true
+end
+

data/config.ru

@@ -0,0 +1,2 @@
+require File.dirname(__FILE__) + "/lib/jor.rb"
+Rack::Handler::Thin.run(JOR::Server.new(Redis.new(:driver => :hiredis)), :Port => 10902) 

data/jor.gemspec

@@ -0,0 +1,29 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib/', __FILE__)
+$:.unshift lib unless $:.include?(lib)
+
+require 'jor/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "jor"
+
+  gem.authors       = ["Josep M. Pujol"]
+  gem.email         = 'josep@3scale.net'
+
+  gem.description   = %q{JSON on top of Redis}
+  gem.summary       = %q{Storage engine for JSON documents using Redis. It allows fast find operations (index) by any field of the JSON document (ala MongoDB)}
+  gem.homepage      = ""
+
+  gem.files         = `git ls-files`.split($\)
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+  gem.version       = JOR::VERSION
+
+  gem.add_dependency 'hiredis',       	'0.4.5'
+  gem.add_dependency 'redis',         	'3.0.3'
+  gem.add_dependency 'rake'
+  gem.add_dependency 'rack',		'1.5.2'	
+  gem.add_dependency 'rack-test'
+  gem.add_dependency 'thin'
+end
+

data/lib/jor.rb

@@ -0,0 +1,14 @@
+require 'json'
+require 'hiredis'
+require 'redis'
+require 'rack'
+
+require File.dirname(__FILE__) + "/jor/errors"
+require File.dirname(__FILE__) + "/jor/storage"
+require File.dirname(__FILE__) + "/jor/collection"
+require File.dirname(__FILE__) + "/jor/doc"
+require File.dirname(__FILE__) + "/jor/server"
+require File.dirname(__FILE__) + "/jor/version"
+
+module JOR
+end

data/lib/jor/collection.rb

@@ -0,0 +1,449 @@
+module JOR
+  class Collection
+
+    DEFAULT_OPTIONS = {
+      :max_documents => 1000,
+      :raw => false,
+      :only_ids => false,
+      :reversed => false,
+      :excluded_fields_to_index => {}
+    }
+
+    def initialize(storage, name, auto_increment = false)
+      @storage = storage
+      @name = name
+      @auto_increment = auto_increment
+    end
+
+    def name
+      @name
+    end
+
+    def redis
+      @storage.redis
+    end
+
+    def storage
+      @storage
+    end
+
+    def auto_increment?
+      @auto_increment
+    end
+
+    def insert(docs, options = {})
+      raise NotInCollection.new unless name
+      opt = merge_and_symbolize_options(options)
+
+      docs.is_a?(Array) ? docs_list = docs : docs_list = [docs]
+
+      docs_list.each_with_index do |doc, i|
+        if auto_increment?
+          raise DocumentDoesNotNeedId.new(name) unless doc["_id"].nil?
+          doc["_id"] = next_id()
+        else
+          raise DocumentNeedsId.new(name) if doc["_id"].nil?
+        end
+
+        doc["_created_at"] ||= Time.now.to_f
+
+        encd = JSON::generate(doc)
+        paths = Doc.paths("!",doc)
+        id = doc["_id"]
+
+        raise InvalidDocumentId.new(id) if !id.is_a?(Numeric) || id < 0
+
+        if !opt[:excluded_fields_to_index].nil? && opt[:excluded_fields_to_index].size>0
+          excluded_paths = Doc.paths("!",opt[:excluded_fields_to_index])
+          paths = Doc.difference(paths, excluded_paths)
+        end
+
+        redis.watch(doc_key(id))
+        exists = redis.get(doc_key(id))
+
+        if !exists.nil?
+          redis.multi
+          redis.exec
+          raise DocumentIdAlreadyExists.new(id, name)
+        else
+          res = redis.multi do
+            redis.set(doc_key(id),encd)
+            redis.zadd(doc_sset_key(),id,id)
+            paths.each do |path|
+              add_index(path,id)
+            end
+          end
+          raise DocumentIdAlreadyExists.new(id, name) unless exists.nil?
+        end
+      end
+      docs
+    end
+
+    def delete(doc)
+      raise NotInCollection.new unless name
+      ids = find(doc, {:only_ids => true, :max_documents => -1})
+      ids.each do |id|
+        delete_by_id(id)
+      end
+      ids.size
+    end
+
+    def update(doc_dest, doc_source, options = {})
+      raise NotInCollection.new unless name
+      doc_source.delete("_id") unless doc_source["_id"].nil?
+      opt = merge_and_symbolize_options(options)
+
+      doc_source["_updated_at"] = Time.now.to_f
+      paths_all = Doc.paths("!",doc_source)
+      excluded_paths = []
+
+      if !opt[:excluded_fields_to_index].nil? && opt[:excluded_fields_to_index].size>0
+        excluded_paths = Doc.paths("!",opt[:excluded_fields_to_index])
+      end
+
+      paths_to_index = Doc.difference(paths_all, excluded_paths)
+
+      docs = find(doc_dest, {:max_documents => -1})
+
+      results = []
+
+      docs.each do |doc|
+
+        indexes_doc = redis.smembers(idx_set_key(doc["_id"]))
+
+        to_remove = []
+        indexes_doc.each do |index_doc|
+          ## for each index there is, check if it's affected by the new paths
+          path_to_from_index = get_path_to_from_index(index_doc)
+
+          paths_all.each do |path|
+            if path["obj"].kind_of?(NilClass)
+              if path["path_to"]==path_to_from_index
+                to_remove << index_doc
+              elsif path_to_from_index.index("#{path["path_to"]}/")!=nil
+                to_remove << index_doc
+              end
+            else
+              if path["path_to"]==path_to_from_index
+                to_remove << index_doc
+              end
+            end
+          end
+        end
+
+        redis.pipelined do
+          to_remove.each do |index|
+            remove_index(index, doc["_id"])
+            redis.srem(idx_set_key(doc["_id"]),index)
+          end
+        end
+
+        ## now, the indexes that refer to the changed fields are gone
+        new_doc = Doc.deep_merge(doc, doc_source)
+        encd = JSON::generate(new_doc)
+
+        res = redis.multi do
+          redis.set(doc_key(new_doc["_id"]),encd)
+          ## note that it's not paths_all but only the ones who are not excluded
+          paths_to_index.each do |path|
+            add_index(path,new_doc["_id"])
+          end
+        end
+
+        results << new_doc
+
+      end
+
+      return results
+    end
+
+    def count
+      raise NotInCollection.new unless name
+      redis.zcard(doc_sset_key())
+    end
+
+    def find(doc, options = {})
+      raise NotInCollection.new unless name
+      # list of ids of the documents
+      ids = []
+      opt =  merge_and_symbolize_options(options)
+
+      if opt[:max_documents] >= 0
+        num_docs = opt[:max_documents]-1
+      else
+        num_docs = -1
+      end
+
+      ## if doc contains _id it ignores the rest of the doc's fields
+      if !doc["_id"].nil? && !doc["_id"].kind_of?(Hash) && doc.size==1
+        ids << doc["_id"]
+        return [] if opt[:only_ids]==true && redis.get(doc_key(ids.first)).nil?
+      elsif (doc == {})
+        if (opt[:reversed]==true)
+          ids = redis.zrevrange(doc_sset_key(),0,num_docs)
+        else
+          ids = redis.zrange(doc_sset_key(),0,num_docs)
+        end
+        ids.map!(&:to_i)
+        ##ids = redis.smembers(doc_sset_key())
+      else
+        paths = Doc.paths("!",doc)
+        ## for now, consider all logical and
+        paths.each_with_index do |path, i|
+          tmp_res = fetch_ids_by_index(path)
+          if i==0
+            ids = tmp_res
+          else
+            ids = ids & tmp_res
+          end
+        end
+
+        ids.map!(&:to_i)
+        ids.reverse! if opt[:reversed]
+      end
+
+      return [] if ids.nil? || ids.size==0
+
+      ## return only up to max_documents, if max_documents is negative
+      ## return them all (they have already been reversed)
+      if opt[:max_documents] >= 0 && opt[:max_documents] < ids.size
+        ids = ids[0..opt[:max_documents]-1]
+      end
+
+      ## return only the ids, it saves fetching the JSON string from
+      ## redis and decoding it
+      return ids if (opt[:only_ids]==true)
+
+      results = redis.pipelined do
+        ids.each do |id|
+          redis.get(doc_key(id))
+        end
+      end
+
+      ## remove nils
+      results.delete_if {|i| i == nil}
+
+      ## return the results JSON encoded (raw), many times you do not need the
+      ## object but only the JSON string
+      return results if (opt[:raw]==true)
+
+      results.map! { |item| JSON::parse(item) }
+      return results
+    end
+
+    def last_id
+      if auto_increment?
+        val = redis.get("#{Storage::NAMESPACE}/#{name}/next_id")
+        return val.to_i unless val.nil?
+        return 0
+      else
+        val = redis.zrevrange(doc_sset_key(),0,0)
+        return 0 if val.nil? || val.size==0
+        return val.first.to_i
+      end
+    end
+
+    def next_id
+      redis.incrby("#{Storage::NAMESPACE}/#{name}/next_id",1)
+    end
+
+    def indexes(id)
+      indexes = redis.smembers(idx_set_key(id))
+      res = []
+      indexes.each do |ind|
+        v = ind.split("!")
+        str = v[1..v.size].join("!")
+
+        v = str.split("/")
+        type = v[v.size-2]
+        path = v[0..v.size-3].join("/")
+
+        if path!=""
+          value_tmp = v[v.size-1]
+          w = value_tmp.split("_")
+          value = w[0..w.size-2].join("_")
+          ## returns the value of a Numeric as string, to cast to float or int, cannot
+          ## be derived from type because it Numeric
+          res << {"path" => path, "obj" => type, "value" => value}
+        end
+      end
+      return res
+    end
+
+    protected
+
+    def merge_and_symbolize_options(options = {})
+      DEFAULT_OPTIONS.merge(options.inject({}){|memo,(k,v)| memo[k.to_sym] = v; memo})
+    end
+
+    def find_docs(doc)
+      return [doc["_id"]] if !doc["_id"].nil?
+
+      ids = []
+      paths = Doc.paths("!",doc)
+
+      ## for now, consider all logical and
+      paths.each_with_index do |path, i|
+        tmp_res = fetch_ids_by_index(path)
+        if i==0
+          ids = tmp_res
+        else
+          ids = ids & tmp_res
+        end
+      end
+      ids
+    end
+
+    def check_selectors(sel)
+      Storage::SELECTORS.each do |type, set|
+        istrue = true
+        sel.each do |s|
+          istrue = istrue && set.member?(s)
+        end
+        return type if istrue
+      end
+      raise IncompatibleSelectors.new(selectors)
+    end
+
+    def find_type(obj)
+      [String, Numeric, TrueClass, FalseClass].each do |type|
+        return type if obj.kind_of? type
+      end
+      raise TypeNotSupported.new(obj.class)
+    end
+
+    def fetch_ids_by_index(path)
+
+      if path["selector"]==true
+        type = check_selectors(path["obj"].keys)
+        if type == :compare
+          key = idx_key(path["path_to"], find_type(path["obj"].values.first))
+
+          rmin = "-inf"
+          rmin = path["obj"]["$gte"] unless path["obj"]["$gte"].nil?
+          rmin = "(#{path["obj"]["$gt"]}" unless path["obj"]["$gt"].nil?
+
+          rmax = "+inf"
+          rmax = path["obj"]["$lte"] unless path["obj"]["$lte"].nil?
+          rmax = "(#{path["obj"]["$lt"]}" unless path["obj"]["$lt"].nil?
+
+          ##ZRANGEBYSCORE zset (5 (10 : 5 < x < 10
+          return redis.zrangebyscore(key,rmin,rmax)
+        elsif type == :sets
+          if path["obj"]["$in"]
+            target = path["obj"]["$in"]
+            join_set = []
+            target.each do |item|
+              join_set = join_set | redis.smembers(idx_key(path["path_to"], find_type(item), item))
+            end
+            return join_set.sort
+          elsif path["obj"]["$all"]
+            join_set = []
+            target = path["obj"]["$all"]
+            target.each do |item|
+              if join_set.size==0
+                join_set = redis.smembers(idx_key(path["path_to"], find_type(item), item))
+              else
+                join_set = join_set & redis.smembers(idx_key(path["path_to"], find_type(item), item))
+              end
+              return [] if (join_set.nil? || join_set.size==0)
+            end
+            return join_set.sort
+          elsif !path["obj"]["$not"].nil?
+            target = path["obj"]["$not"]
+            target = [target] unless target.kind_of?(Array)
+            join_set = []
+            target.each do |item|
+              join_set = join_set | redis.smembers(idx_key(path["path_to"], find_type(item), item))
+            end
+            return (redis.zrange(doc_sset_key(),0,-1) - join_set).sort
+          end
+        end
+      end
+
+      [String, Numeric, TrueClass, FalseClass].each do |type|
+        if path["obj"].kind_of?(type)
+          return redis.smembers(idx_key(path["path_to"], type, path["obj"])).sort
+        end
+      end
+      path["obj"].kind_of?(NilClass) ? [] : (raise TypeNotSupported.new(path["obj"].class))
+    end
+
+    def delete_by_id(id)
+      indexes = redis.smembers(idx_set_key(id))
+      v = redis.pipelined do
+        indexes.each do |index|
+          remove_index(index, id)
+        end
+        redis.del(idx_set_key(id))
+        redis.zrem(doc_sset_key(),id)
+        redis.del(doc_key(id))
+      end
+    end
+
+    def remove_index(index, id)
+      v = index.split("_")
+      key = v[0..v.size-2].join("_")
+      if v.last=="srem"
+        redis.srem(key, id)
+      elsif v.last=="zrem"
+        redis.zrem(key, id)
+      else
+        raise UnknownIndex.new(index,id)
+      end
+    end
+
+    def add_index(path, id)
+      [String, Numeric, TrueClass, FalseClass].each do |type|
+        if path["obj"].kind_of?(type)
+          key = idx_key(path["path_to"], type, path["obj"])
+          redis.sadd(key, id)
+          redis.sadd(idx_set_key(id), "#{key}_srem")
+          if type==Numeric
+            key = idx_key(path["path_to"], type)
+            redis.zadd(key, path["obj"], id)
+            redis.sadd(idx_set_key(id), "#{key}_zrem")
+          end
+          return true
+        end
+      end
+      path["obj"].kind_of?(NilClass) ? true : (raise TypeNotSupported.new(path["obj"].class))
+    end
+
+    def get_path_to_from_index(index)
+      ## this traverses all indexes, Numeric fields have two actual indexes that do not
+      ## appear on the indexes method
+      ini_pos = index.index("/!")
+      ini_pos += 1
+
+      end_pos = index.index("/String")
+      end_pos = index.index("/Numeric") if end_pos.nil?
+      end_pos = index.index("/TrueClass") if end_pos.nil?
+      end_pos = index.index("/FalseClass") if end_pos.nil?
+
+      raise CouldNotFindPathToFromIndex.new(index) if ini_pos.nil? || end_pos.nil?
+      end_pos -= 1
+
+      return index[ini_pos..end_pos]
+    end
+
+    def idx_key(path_to, type, obj = nil)
+      tmp = ""
+      tmp = "/#{obj}" unless obj.nil?
+      "#{Storage::NAMESPACE}/#{name}/idx/#{path_to}/#{type}#{tmp}"
+    end
+
+    def idx_set_key(id)
+      "#{Storage::NAMESPACE}/#{name}/sidx/#{id}"
+    end
+
+    def doc_key(id)
+      "#{Storage::NAMESPACE}/#{name}/docs/#{id}"
+    end
+
+    def doc_sset_key()
+      "#{Storage::NAMESPACE}/#{name}/ssdocs"
+    end
+
+  end
+end