strokedb 0.0.2

data/CONTRIBUTORS

@@ -0,0 +1,7 @@
+Adrian Madrid <aemadrid@gmail.com>
+Aman Gupta <aman@tmm1.net>
+Claudio Perez Gamayo <crossblaim@gmail.com>
+elliottcable.name <strokedb@elliottcable.com>
+Joshua Miller <elefantstn@gmail.com>
+Oleg Dashevskii <be9@be9.ru>
+Michael Klishin (novemberain.com) <michael.s.klishin@gmail.com>

data/CREDITS

@@ -0,0 +1,13 @@
+
+MANY THANKS TO
+(for knowledge and aspiration)
+
+1) Andrew S. Tanenbaum, for book "Distributed Systems".
+2) William Pugh for skiplists.
+3) Xin Dong & Alon Halevy for the great article "Indexing Dataspaces".
+4) Linus Torvalds & Junio C. Hamano for the Git version control system.
+5) Damien Katz for the CouchDB.
+6) Yukihiro Matsumoto for the Ruby programming language.
+7) Dr. Leslie Lamport for timestamps and signatures.
+8) Victor Sovetov for years of talking with Yurii about databases and metaframes.
+9) Konstantin Olenin for talks with Oleg about distributed systems and algorithms.

data/README

@@ -0,0 +1,44 @@
+StrokeDB is a distributed document-oriented database engine. 
+Main features are complete decentralization, JSON object format, 
+metadocuments, integration with runtime (it is just a ruby library). 
+
+= Installing StrokeDB
+
+=== Requirements
+
+You need to install few gems in order to run StrokeDB:
+
+	$ sudo gem install diff-lcs uuidtools json
+
+*Note*: There is no need in <tt>uuidtools</tt> if you use JRuby
+
+=== Getting source code
+
+	$ git clone git://gitorious.org/strokedb/mainline.git strokedb
+or
+	$ git clone http://git.gitorious.org/strokedb/mainline.git strokedb
+
+=== Installing rubygem
+
+There is no gem for StrokeDB (yet). As soon as we'll release 0.1, it will become available this way, too.
+	
+=== <i>(Optional) Running test suite</i>
+
+	$ cd strokedb/strokedb-ruby
+	$ rake ci
+	$ rake jci # for jruby, jruby should be in PATH
+	
+
+= Starting points
+
+One of the most important concepts of StrokeDB is a StrokeDB::Document.
+
+
+= Some benchmarks
+
+	$ rake bench
+
+=AUTHORS
+
+* Yurii Rashkovskii <yrashk@issuesdone.com>
+* Oleg Andreev <oleganza@gmail.com>

data/bin/sdbc

@@ -0,0 +1,2 @@
+#! /usr/bin/env ruby
+load File.dirname(__FILE__) + '/../script/console'

data/lib/config/config.rb

@@ -0,0 +1,161 @@
+module StrokeDB
+  # errors raised in the process of configuration
+
+  class UnknownStorageTypeError < Exception; end
+  class UnknownIndexTypeError   < Exception; end
+  class UnknownStoreTypeError   < Exception; end
+
+  class Config
+    #
+    # Load config from file, probably making it the default one
+    #
+    def Config.load(filename, default = false)
+      build(JSON.parse(IO.read(filename)).merge(:default => default))
+    end
+
+    #
+    # Build the config from given options.
+    #
+    # Supported options are:
+    #
+    #   :default - if set to true, config becomes the default one.
+    #   :storages - must be an array of storage types.
+    #               Appropriate storages will be initialized and chained
+    #               together. Defaults to [:memory_chunk, :file_chunk]
+    #   :index_storages - index storages. Defaults to [:inverted_list_file].
+    #   :index - index type. Defaults to :inverted_list. 
+    #   :base_path - if set, specifies the path for storages. Otherwise,
+    #                current directory is used.
+    #   :store - store type to use. Defaults to :skiplist.
+    #   :store_options - options passed to the created store
+
+    def Config.build(opts={})
+      opts = opts.stringify_keys
+      
+      config = new(opts['default'])
+      storages = opts['storages'] || [:memory_chunk, :file_chunk]
+
+      base_path = opts['base_path'] || './'
+
+      add_storage = lambda do |name| 
+        config.add_storage(name, name, :path => File.join(base_path, name.to_s))
+      end
+
+      ### setup document storages ###
+      
+      initialized_storages = storages.map(&add_storage)       
+      config.chain(*storages) if storages.size >= 2
+
+      initialized_storages.each_consecutive_pair do |cur, nxt|
+        # next storage is authoritative for each storage
+        cur.authoritative_source = nxt
+      end
+      
+      ### setup index storages and indexes ###
+
+      index_storages = opts['index_storages'] || [:inverted_list_file]
+      index_storages.each(&add_storage)
+
+      config.add_index(:default, opts['index'] || :inverted_list, index_storages.first)
+
+      config.add_store(:default, opts['store'] || :skiplist, 
+                       { :storage => storages.first }.merge(opts['store_options'] || {}))
+
+      ### save config ###
+
+      config.build_config = opts.except('default')
+      
+      FileUtils.mkdir_p base_path
+      File.open(File.join(base_path,'config'), "w+") do |file|
+        file.write config.build_config.to_json
+      end
+
+      config
+    end
+      
+    attr_accessor :build_config
+    attr_reader :storages, :indexes, :stores
+
+    def initialize(default = false)
+      @storages, @indexes, @stores = {}, {}, {}
+
+      ::StrokeDB.default_config = self if default
+    end
+ 
+    def [](name)
+      @storages[name] || @indexes[name] || nil
+    end
+    
+    def add_storage(key, type, *args)
+      @storages[key] = constantize(:storage, type).new(*args)
+    end
+    
+    def chain_storages(*args)
+      raise ArgumentError, "Not enough storages to chain storages" unless args.size >= 2
+
+      args.map {|x| @storages[x] || raise("Missing storage #{x}") }.each_consecutive_pair do |cur, nxt|
+        cur.add_chained_storage! nxt
+      end
+    end
+    
+    alias :chain :chain_storages
+    
+    def add_index(key, type, storage_key, store_key = nil)
+      @indexes[key] = constantize(:index, type).new(@storages[storage_key])
+      @indexes[key].document_store = @stores[store_key] if store_key
+    end
+    
+    def add_store(key, type, options = {})
+      options[:storage] = @storages[options[:storage] || :default]
+      raise "Missing storage for store #{key}" unless options[:storage]
+
+      options[:index] ||= @indexes[options[:index] || :default]
+
+      store_instance = constantize(:store, type).new(options)
+
+      if options[:index]
+        options[:index].document_store = store_instance
+      end
+
+      @stores[key] = store_instance
+    end
+    
+    private
+    
+    def constantize(name,type)
+      StrokeDB.const_get type_fullname(name,type)
+    rescue 
+      exception = StrokeDB.const_get("Unknown#{name.to_s.camelize}TypeError")
+      raise exception, "Unable to load #{name} type #{type}"
+    end
+    
+    def type_fullname(type, name)
+      "#{name.to_s.camelize}#{type.to_s.camelize}"
+    end
+  end
+  
+  class <<self
+    def use_perthread_default_config!
+      class <<self
+        def default_config
+          Thread.current['StrokeDB.default_config']
+        end
+        def default_config=(config)
+          Thread.current['StrokeDB.default_config'] = config
+        end
+      end
+    end
+    def use_global_default_config!
+      class <<self
+        def default_config
+          $strokedb_default_config
+        end
+        def default_config=(config)
+          $strokedb_default_config = config
+        end
+      end
+    end
+  end
+end
+
+StrokeDB.use_perthread_default_config!

data/lib/data_structures/inverted_list.rb

@@ -0,0 +1,297 @@
+module StrokeDB
+  class InvertedList
+  	include Enumerable
+  	
+  	SEPARATOR  = "\x01" 
+  	TERMINATOR = "\x02"
+  	
+  	attr_accessor :default, :head, :tail, :cut_level
+
+  	def initialize(cut_level = nil)
+  		@cut_level = cut_level
+  		@head = HeadNode.new
+  		@tail = TailNode.new
+  		@head.forward[0] = @tail
+  	end
+
+  	def insert(slots, data, __cheaters_level = nil)
+  	  slots.each do |key, value|
+  	    value = value.to_s
+  	    key = key.to_s
+  	    prefix = value + SEPARATOR + key + TERMINATOR
+  	    insert_attribute(prefix, data, __cheaters_level)
+	    end
+  	end
+
+  	def insert_attribute(key, value, __cheaters_level = nil)
+  	  @size_cache = nil
+  	  update = Array.new(@head.level)
+  	  x = @head
+      @head.level.downto(1) do |i|
+	      x = x.forward[i-1] while x.forward[i-1] < key
+	      update[i-1] = x
+	    end
+  	  x = x.forward[0]
+  	  if x.key == key
+  	    x.values.push value
+  	  else
+  	    newlevel = __cheaters_level || random_level
+  	    newlevel = 1 if empty?
+  	    if newlevel > @head.level 
+  	      (@head.level + 1).upto(newlevel) do |i|
+  	        update[i-1] = @head
+          end
+        end
+
+        x = Node.new(newlevel, key, value)
+
+        if cut?(newlevel, update[0])
+          return new_chunks!(x, update)
+        else
+          newlevel.times do |i|
+            x.forward[i] = update[i].forward[i] || @tail
+            update[i].forward[i] = x
+          end
+        end
+      end
+  		return self
+  	end
+  	
+  	
+  	def delete(slots, data)
+  	  slots.each do |key, value|
+  	    value = value.to_s
+  	    key = key.to_s
+  	    prefix = value + SEPARATOR + key + TERMINATOR
+  	    delete_attribute(prefix, data)
+	    end
+  	end
+  	
+  	def delete_attribute(key, value)
+  	  @size_cache = nil
+  	  update = Array.new(@head.level)
+  	  x = @head
+      @head.level.downto(1) do |i|
+	      x = x.forward[i-1] while x.forward[i-1] < key
+	      update[i-1] = x
+	    end
+  	  x = x.forward[0]
+  	  if x.key == key
+  	    x.values.delete value
+  	    value
+  	  else
+  	    nil
+      end
+  	end
+  	  	
+
+    # Finders
+    
+    def find(*args)
+  	  q = PointQuery.new(*args)
+  	  total = Set.new
+  	  first_pass = true
+  	  q.slots.each do |key, value|
+  	    results = []
+  	    key = key.to_s
+  	    value = value.to_s
+  	    prefix = value + SEPARATOR + key + TERMINATOR
+  	    node = find_node(prefix)
+  	    results = node.values if node
+  	    total = (first_pass ? results.to_set : (total & results))
+  	    first_pass = false
+	    end
+	    total
+  	end
+  	
+    def find_node(key)
+      x = @head
+      @head.level.downto(1) do |i|
+  	    x = x.forward[i-1] while x.forward[i-1] < key
+  	  end
+  	  x = x.forward[0]
+	    return (x.key && yield(x.key, key) ? x : nil) if block_given?
+  	  return x if x.key == key
+  	  nil
+    end
+
+    def first_node
+      @head.forward[0]
+    end
+
+    def size
+  		@size_cache ||= inject(0){|c,k| c + 1}
+  	end
+
+  	def empty?
+  	  @head.forward[0] == @tail
+  	end
+
+  	# Returns a string representation of the Skiplist.
+  	def to_s
+  		"#<#{self.class.name} " + 
+  		[@head.to_s, map{|node| node.to_s }, @tail.to_s].flatten.join(', ') +
+  		">"
+  	end
+  	def to_s_levels
+  		"#<#{self.class.name}:levels " + 
+  		[@head.to_s, map{|node| node.level.to_s }, @tail.to_s].flatten.join(', ') +
+  		">"
+  	end
+  	
+  	def debug_dump
+  	  s = ""
+  	  each do |n|
+  	    s << "#{n.key.inspect}: #{n.values.inspect}\n"
+      end
+      s
+	  end
+
+ 	  def each
+  	  n = @head.forward[0]
+  	  until TailNode === n
+  	    yield n
+  	    n = n.forward[0]
+      end
+  	end
+
+  private
+
+    # 1/E is a fastest search value
+  	PROBABILITY = 1/Math::E
+  	MAX_LEVEL   = 32
+
+  	def random_level
+  		l = 1
+  		l += 1 while rand < PROBABILITY && l < MAX_LEVEL
+  		return l
+  	end
+
+  	def cut?(l, prev)
+    	@cut_level && !empty? && l >= @cut_level && prev != @head
+  	end
+
+  	def new_chunks!(newnode, update)
+  	  # Transposed picture:
+  	  #
+  	  # head level 8:     - - - - - - - -
+  	  # update.size 8:    - - - - - - - -
+  	  # ...
+  	  # newnode.level 5:  - - - - -
+  	  # cut level 3:      - - - 
+   	  # regular node:     - 
+  	  # regular node:     - -
+  	  # ...                        
+  	  # tail node:        T T T T T T T T   
+  	  #           refs:   A B C D E F G H 
+  	  #
+  	  # How to cut?
+  	  #
+      # 0) tail1 = TailNode.new; list2 = Skiplist.new
+  	  # 1) newnode.{A, B, C, D, E} := update{A,B,C,D,E}.forward
+  	  # 2) update.{all} := tail1 (for current chunk)
+  	  # 3) list2.head.{A, B, C, D, E} = new_node.{A, B, C, D, E}
+  	  # 4) tail1.next_list = list2
+
+  	  list2 = Skiplist.new({}, @default, @cut_level)
+  	  tail1 = TailNode.new
+
+  	  newnode.level.times do |i|
+  	    # add '|| @tail' because update[i] may be head of a lower level
+  	    # without forward ref to tail.
+  	    newnode.forward[i] = update[i].forward[i] || @tail
+  	    list2.head.forward[i] = newnode
+	    end
+	    @head.level.times do |i|
+	      update[i].forward[i] = tail1
+        end
+	    tail1.next_list = list2
+  	  # return the current chunk and the next chunk
+  	  return self, list2
+  	end
+
+  	class Node
+  		attr_accessor :key, :values, :forward
+  		attr_accessor :_serialized_index
+  		def initialize(level, key, value)
+  			@key, @values = key, [value]
+  			@forward = Array.new(level)
+  		end
+  		# this is called when node is thrown out of the list
+  		# note, that node.value is called immediately after node.free
+  		def free(list)
+  		  # do nothing
+  		end
+  		def level
+  		  @forward.size
+  		end
+  		def <(key)
+  		  @key < key
+  	  end
+  	  def <=(key)
+  		  @key <= key
+  	  end
+  	  def next
+  	    forward[0]
+  	  end
+  	  def to_s
+  	    "[#{level}]#{@key}: #{@values.inspect}"
+  	  end
+  	end
+
+  	class HeadNode < Node
+  		def initialize
+  			super 1, nil, nil
+  		end
+  		def <(key)
+  		  true
+  	  end
+  	  def <=(key)
+  		  true
+  	  end
+      def to_s
+  	    "head(#{level})"
+  	  end
+  	end
+
+  	# also proxy-to-next-chunk node
+  	class TailNode < Node
+  	  attr_accessor :next_list
+  		def initialize
+  			super 1, nil, nil
+  		end
+  		def <(key)
+  		  false
+  	  end
+  	  def <=(key)
+  		  false
+  	  end
+  	  def to_s
+  	    "tail(#{level})"
+  	  end
+  	end
+  
+  	def debug(msg)
+      if block_given?
+        begin
+          out = []
+          out << "\n\n---- Start of #{msg} -----"
+          yield(out)
+          return
+        rescue => e
+          puts out.join("\n")
+          puts "---- End of #{msg}: exception! -----"
+          puts e
+          puts e.backtrace.join("\n") rescue nil
+          puts "----"
+          raise e
+        end
+      else
+        puts "IL DEBUG: #{msg}" if ENV['DEBUG']
+      end
+    end
+    def debug_header
+      puts "\n==========================================\n" if ENV['DEBUG']
+    end
+	end
+end

data/lib/data_structures/point_query.rb

@@ -0,0 +1,24 @@
+module StrokeDB
+  # PointQuery is used to perform navigation to a single multidimensinal point.
+  # Initializer accepts a hash of slots. Slots may have such value types: 
+  #   "string"           scalar string value
+  #   3.1415 (numeric)   numeric value
+  #   :L                 lowest value
+  #   :H                 highest value
+  #
+  # Example:
+  #   PointQuery.new(:meta   => 'Article', 
+  #                  :author => 'Oleg Andreev', 
+  #                  :date   => :last)
+  #
+  class PointQuery
+    attr_reader :slots
+
+    def initialize(slots)
+      @slots = {}
+      slots.each do |k, v|
+        @slots[k.to_optimized_raw] = v.to_optimized_raw
+      end
+    end
+  end
+end