strokedb 0.0.2.1 → 0.0.2.2

data/lib/strokedb/volumes/distributed_pointer.rb

@@ -0,0 +1,43 @@
+module StrokeDB
+  # DP is for pointing to DataVolume item. It can address zillions
+  # of DataVolumes, 4 GB each.
+  #
+  # DP is stored as a 160 bit bytestring. First 128 bits is a volume UUID.
+  # Next 32 bits is an offset in a volume (unsigned long). Hence, the limit
+  # for DataVolume size: max 4 Gb. Actually, real-world applications would 
+  # have relatively small datavolumes (about 64 Mb).
+  class DistributedPointer
+    attr_accessor :volume_uuid, :offset
+    
+    # Initialize pointer with given components.
+    # * uuid UUID (either raw or formatted)
+    # * offset is a positive integer
+    #
+    def initialize(uuid, offset)
+      @volume_uuid = uuid.to_raw_uuid
+      @offset      = offset
+    end
+    
+    # Creates a pointer object using binary string.
+    #
+    def self.unpack(string160bit)
+      new(string160bit[0, 16], string160bit[16, 4].unpack("L")[0])
+    end
+    
+    # Converts pointer object to it's string representation.
+    #
+    def pack
+      @volume_uuid + [@offset].pack("L")
+    end
+    
+    def self.pack(uuid,offset)
+      uuid.to_raw_uuid + [offset].pack("L")
+    end
+    
+    def inspect #:nodoc:
+      "#<DistributedPointer #{@volume_uuid.to_formatted_uuid}:#{@offset}>"
+    end
+    
+    alias :to_s :inspect
+  end
+end

data/lib/strokedb/volumes/fixed_length_skiplist_volume.rb

@@ -0,0 +1,109 @@
+module StrokeDB
+  class FixedLengthSkiplistVolume < SimpleSkiplist
+
+    def initialize(options = {})
+      @options = options.stringify_keys
+      @volume = MapVolume.new(:record_size => (@options['maxlevel']||DEFAULT_MAXLEVEL) * 4 + 1 +
+      @options['key_length'] + @options['value_length'], :path => @options['path'])
+      @nodes = {}
+      super(nil,:maxlevel => @options['maxlevel'], :probability => @options['probability'])
+    end
+
+    def key_length
+      @options['key_length']
+    end
+
+    def value_length
+      @options['value_length']
+    end
+
+    def path
+      @options['path']
+    end
+
+    def close!
+      @volume.close!
+    end
+
+    def inspect
+      "#<StrokeDB::FixedLengthSkiplistVolume:0x#{object_id.to_s(16)} path: #{path} key_length: #{key_length} value_length: #{value_length}"
+    end
+
+    private
+
+    # SimpleSkiplist overrides
+
+
+    def node_next(x, level)
+      if node = x[0][level]
+        read_node(node[-1])
+      else
+        nil
+      end
+    end
+
+    def node_set_value!(x, value)
+      x[-2] = value
+      save_node!(x)
+    end
+
+    def node_insert_after!(x, prev, level)
+      x[0][level] = prev[0][level]
+      prev[0][level] = save_node!(x)
+      save_node!(prev)
+    end
+
+    def new_node(level, key, value, __pos = -1)
+      [
+        [nil]*level,
+        key,
+        value,
+        __pos
+      ]
+    end
+
+    def new_head
+      unless @volume.available?(0)
+        read_node(0)
+      else
+        _head = new_node(@maxlevel, "\x00" * key_length, "\x00" * value_length)
+        save_node!(_head)
+      end
+    end
+
+    def save_node!(node)
+      node_levels = node[0].map{|v| v.nil? ? -1 : v[-1]}
+      packed = ([maxlevel]+node_levels).pack("CN#{node_levels.size}")
+      key = node[-3]
+      value = node[-2]
+      
+      if (szd = maxlevel - node_levels.size) > 0
+        packed += "\xff\xff\xff\xff"*szd
+      end
+
+      if node[-1] == -1 # unsaved
+        node[-1] = @volume.insert!(packed + key + value)
+      else
+        @volume.write!(node[-1],packed + key + value)
+      end
+      @head = node if node[-1] == 0
+      node
+    end
+
+    def read_node(position)
+      _node = @volume.read(position)
+      level = _node[0,1].unpack('C')[0]
+      links = _node[1,maxlevel*4].unpack("N#{level}")
+      node = [
+        LazyMappingArray.new(links).map_with do |v| 
+          v == 4294967295 ? nil : read_node(v)
+        end.unmap_with {|v| v[-1] },
+        _node[maxlevel*4 + 1,key_length], 
+        _node[maxlevel*4 + 1 + key_length, value_length],
+        position
+      ]
+    end
+
+
+  end
+end

data/lib/strokedb/volumes/map_volume.rb

@@ -0,0 +1,268 @@
+require 'readbytes'
+module StrokeDB
+
+  class MapVolume
+
+    HEADER_SIZE = 512
+    MAGIC_SIGNATURE = "\x11\x12\x19\x81"
+    VERSION = "00"
+
+    attr_accessor :first_available_position
+
+    def initialize(options = {})
+      @options = options.stringify_keys
+      initialize_file
+    end
+
+    def insert!(record)
+      position = find_first_available_position
+      write!(position,record)
+    end
+
+    def elastic_insert!(record)
+      position = find_first_available_chunk_position((record.size + 4) % 8)
+      elastic_write!(position,record)
+    end
+
+    def write!(position,record)
+      raise InvalidRecordSizeError if record.size != record_size
+      decrement_available_capacity!(position)
+      write_at_position!(position,record)
+      position
+    end
+
+    def elastic_write!(position,record)
+      decrement_available_chunk!(position,(record.size + 4) % 8)
+      elastic_write_at_position!(position,record)
+      position
+    end
+
+    def read(position)
+      raise InvalidRecordPositionError if available?(position)
+      read_at_position(position)
+    end
+
+    def elastic_read(position)
+      @data_file.seek(position*record_size)
+      size = @data_file.read(4).unpack('N').first
+      @data_file.read(size)
+    end
+
+    def delete!(position)
+      self.first_available_position = -1
+      increment_available_capacity!(position)
+    end
+
+    def available?(position)
+      read_map_byte(position) & (1 << (position % 8)) == 0
+    end
+
+    def empty?
+      read_map == "\x00" * map_size
+    end
+
+    def record_size
+      @options['record_size']
+    end
+
+    def bitmap_extension_pace
+      @options['bitmap_extension_pace']||8192
+    end
+
+    def path
+      @options['path']
+    end
+
+    def close!
+      @bitmap_file.close
+      @data_file.close
+    end
+
+    def map_size
+      return @map_size if @map_size
+      pos = @bitmap_file.pos
+      @bitmap_file.seek(0,IO::SEEK_END)
+      size = @bitmap_file.pos - HEADER_SIZE
+      @bitmap_file.seek(pos)
+      @map_size = size
+    end
+
+
+    private
+
+    def initialize_file
+      FileUtils.mkdir_p path
+      bitmap_path = File.join(path,'bitmap')
+      data_path = File.join(path,'data')
+      unless File.exists?(bitmap_path)
+        @bitmap_file = File.new(bitmap_path,'w+')
+        @first_available_position = 0
+        initialize_file_header!
+        initialize_file_map!
+      else
+        @bitmap_file = File.new(bitmap_path,'r+')
+        @first_available_position = -1
+        read_file_header
+        initialize_file_header!
+      end
+      unless File.exists?(data_path)
+        @data_file = File.new(data_path,'w+')
+      else
+        @data_file = File.new(data_path,'r+')
+      end
+      read_map
+    end
+
+    def read_file_header
+      @bitmap_file.seek(0)
+      begin
+        header = @bitmap_file.readbytes(HEADER_SIZE)
+      rescue TruncatedDataError
+        raise InvalidMapVolumeError
+      end
+      raise InvalidMapVolumeError unless header[0,4] = MAGIC_SIGNATURE
+      @options['record_size'] = header[6,4].unpack("N").first
+      @first_available_position = (pos = header[10,4].unpack("N").first) == 4294967295 ? -1 : pos
+    end
+
+    def initialize_file_header!
+      header = "\xff"*HEADER_SIZE
+      header[0,4] = MAGIC_SIGNATURE
+      header[4,2] = VERSION
+      header[6,4] = [record_size].pack("N")
+      header[10,4] = [first_available_position].pack("N")
+      @bitmap_file.seek(0)
+      @bitmap_file.write(header)
+    end
+
+    def update_file_header!
+      @bitmap_file.seek(10)
+      @bitmap_file.write([first_available_position].pack("N"))
+    end
+
+    def initialize_file_map!
+      extend_map
+    end
+
+
+    def read_map
+      return @bitmap if @bitmap
+      @bitmap_file.seek(HEADER_SIZE)
+      @bitmap = @bitmap_file.read(map_size)
+    end
+
+    def find_first_available_position
+      unless first_available_position == -1
+        first_available_position
+      else
+        byte_num = 0
+        byte = nil
+        read_map.each_byte do |v|  
+          if v != 255
+            byte = v
+            break
+          else
+            byte_num += 1 
+            false
+          end
+        end
+        if byte 
+          byte_offset = byte.to_s(2).ljust(8,'0').index('0')
+          return byte_num*8 + byte_offset
+        end
+        nil
+      end
+    end
+
+    def find_first_available_chunk_position(chunk)
+      unless position = read_map.index("\x00" * chunk)
+        position = map_size * 8
+        extend_map
+      end
+      position
+    end
+
+    def decrement_available_capacity!(position)
+      byte = read_map_byte(position)
+      mask = (1 << (position % 8))
+      return unless byte & mask  == 0
+
+      write_map_byte(position, byte | mask)
+
+      if read_map_byte(position + 1) == 255
+        @first_available_position = -1
+      else
+        @first_available_position = position + 1
+      end
+
+      update_file_header!
+    end
+
+    def decrement_available_chunk!(position,length)
+      @bitmap_file.seek(HEADER_SIZE + (position % 8))
+      update ="\xff" * (length+1)
+      @bitmap_file.write(update)
+      @first_available_position = -1
+      @bitmap[position%8,length+1] = update
+      update_file_header!
+    end
+
+
+    def increment_available_capacity!(position)
+      update_map_byte!(position) {|byte| byte & (255 ^ (1 << (position % 8))) }
+      update_file_header!
+    end
+
+    def update_map_byte!(position)
+      byte = yield(read_map_byte(position))
+      write_map_byte(position,byte)
+    end
+
+    def read_map_byte(position)
+      extend_map if map_size*8 <= position # TODO: spec it
+      read_map[position/8]
+      # @bitmap_file.seek(HEADER_SIZE + position/8)
+      # @bitmap_file.read(1).unpack('C').first # in Ruby 1.8 we can also do [0] instead of unpack
+    end
+
+    def write_map_byte(position,byte)
+      @bitmap_file.seek(HEADER_SIZE + position/8)
+      @bitmap_file.write([byte].pack('C'))
+      @bitmap[position/8] = byte
+    end
+
+    def write_at_position!(position,record)
+      @data_file.seek(position*record_size)
+      @data_file.write(record)
+    end
+
+    def elastic_write_at_position!(position,record)
+      @data_file.seek(position*record_size)
+      @data_file.write([record.size].pack('N') + record)
+    end
+
+    def read_at_position(position)
+      @data_file.seek(position*record_size)
+      @data_file.read(record_size)
+    end
+
+    def extend_map
+      pos = @bitmap_file.pos
+      @bitmap_file.truncate(pos + bitmap_extension_pace)
+      # @bitmap_file.seek(pos)
+      @bitmap += "\x00"*bitmap_extension_pace if @bitmap
+      @map_size = nil
+    end
+
+  end
+
+  class InvalidRecordSizeError < Exception
+  end
+
+  class InvalidRecordPositionError < Exception
+  end
+
+  class InvalidMapVolumeError < Exception
+  end
+
+end

data/meta/MANIFEST

@@ -0,0 +1,175 @@
+bench.html
+bin/strokedb
+examples/movies.rb
+examples/movies2.rb
+examples/strokewiki/README
+examples/strokewiki/view/edit.xhtml
+examples/strokewiki/view/new.xhtml
+examples/strokewiki/view/pages.xhtml
+examples/strokewiki/view/show.xhtml
+examples/strokewiki/view/versions.xhtml
+examples/strokewiki/wiki.rb
+examples/todo.rb
+lib/strokedb/config.rb
+lib/strokedb/console.rb
+lib/strokedb/core_ext/blank.rb
+lib/strokedb/core_ext/enumerable.rb
+lib/strokedb/core_ext/fixnum.rb
+lib/strokedb/core_ext/float.rb
+lib/strokedb/core_ext/hash.rb
+lib/strokedb/core_ext/infinity.rb
+lib/strokedb/core_ext/kernel.rb
+lib/strokedb/core_ext/object.rb
+lib/strokedb/core_ext/string.rb
+lib/strokedb/core_ext/symbol.rb
+lib/strokedb/core_ext.rb
+lib/strokedb/data_structures/chunked_skiplist.rb
+lib/strokedb/data_structures/inverted_list.rb
+lib/strokedb/data_structures/point_query.rb
+lib/strokedb/data_structures/simple_skiplist.rb
+lib/strokedb/data_structures/skiplist.rb
+lib/strokedb/data_structures.rb
+lib/strokedb/document/callback.rb
+lib/strokedb/document/delete.rb
+lib/strokedb/document/dsl/associations.rb
+lib/strokedb/document/dsl/coercions.rb
+lib/strokedb/document/dsl/meta_dsl.rb
+lib/strokedb/document/dsl/validations.rb
+lib/strokedb/document/dsl/virtualize.rb
+lib/strokedb/document/dsl.rb
+lib/strokedb/document/meta.rb
+lib/strokedb/document/slot.rb
+lib/strokedb/document/util.rb
+lib/strokedb/document/versions.rb
+lib/strokedb/document.rb
+lib/strokedb/index.rb
+lib/strokedb/nsurl.rb
+lib/strokedb/store.rb
+lib/strokedb/stores/chainable_storage.rb
+lib/strokedb/stores/file_storage.rb
+lib/strokedb/stores/inverted_list_file_storage.rb
+lib/strokedb/stores/memory_storage.rb
+lib/strokedb/stores/remote_store.rb
+lib/strokedb/stores.rb
+lib/strokedb/sync/chain_sync.rb
+lib/strokedb/sync/diff/array.rb
+lib/strokedb/sync/diff/default.rb
+lib/strokedb/sync/diff/hash.rb
+lib/strokedb/sync/diff/string.rb
+lib/strokedb/sync/diff.rb
+lib/strokedb/sync/lamport_timestamp.rb
+lib/strokedb/sync/store_sync.rb
+lib/strokedb/sync.rb
+lib/strokedb/transaction.rb
+lib/strokedb/util/attach_dsl.rb
+lib/strokedb/util/blankslate.rb
+lib/strokedb/util/class_optimization.rb
+lib/strokedb/util/inflect.rb
+lib/strokedb/util/java_util.rb
+lib/strokedb/util/lazy_array.rb
+lib/strokedb/util/lazy_mapping_array.rb
+lib/strokedb/util/lazy_mapping_hash.rb
+lib/strokedb/util/serialization.rb
+lib/strokedb/util/uuid.rb
+lib/strokedb/util/xml.rb
+lib/strokedb/util.rb
+lib/strokedb/view.rb
+lib/strokedb/volumes/archive_volume.rb
+lib/strokedb/volumes/block_volume.rb
+lib/strokedb/volumes/distributed_pointer.rb
+lib/strokedb/volumes/fixed_length_skiplist_volume.rb
+lib/strokedb/volumes/map_volume.rb
+lib/strokedb/volumes.rb
+lib/strokedb.rb
+README
+script/console
+spec/integration/remote_store_spec.rb
+spec/integration/search_spec.rb
+spec/integration/spec_helper.rb
+spec/lib/spec_helper.rb
+spec/lib/strokedb/config_spec.rb
+spec/lib/strokedb/core_ext/blank_spec.rb
+spec/lib/strokedb/core_ext/extract_spec.rb
+spec/lib/strokedb/core_ext/float_spec.rb
+spec/lib/strokedb/core_ext/infinity_spec.rb
+spec/lib/strokedb/core_ext/spec_helper.rb
+spec/lib/strokedb/core_ext/string_spec.rb
+spec/lib/strokedb/core_ext/symbol_spec.rb
+spec/lib/strokedb/data_structures/chunked_skiplist_spec.rb
+spec/lib/strokedb/data_structures/inverted_list_spec.rb
+spec/lib/strokedb/data_structures/simple_skiplist_spec.rb
+spec/lib/strokedb/data_structures/skiplist_spec.rb
+spec/lib/strokedb/data_structures/spec_helper.rb
+spec/lib/strokedb/document/associations_spec.rb
+spec/lib/strokedb/document/callbacks_spec.rb
+spec/lib/strokedb/document/coercions_spec.rb
+spec/lib/strokedb/document/document_spec.rb
+spec/lib/strokedb/document/meta_meta_spec.rb
+spec/lib/strokedb/document/meta_spec.rb
+spec/lib/strokedb/document/metaslot_spec.rb
+spec/lib/strokedb/document/slot_spec.rb
+spec/lib/strokedb/document/spec_helper.rb
+spec/lib/strokedb/document/validations_spec.rb
+spec/lib/strokedb/document/virtualize_spec.rb
+spec/lib/strokedb/nsurl_spec.rb
+spec/lib/strokedb/spec_helper.rb
+spec/lib/strokedb/stores/chained_storages_spec.rb
+spec/lib/strokedb/stores/spec_helper.rb
+spec/lib/strokedb/stores/store_spec.rb
+spec/lib/strokedb/stores/transaction_spec.rb
+spec/lib/strokedb/sync/chain_sync_spec.rb
+spec/lib/strokedb/sync/diff_spec.rb
+spec/lib/strokedb/sync/lamport_timestamp_spec.rb
+spec/lib/strokedb/sync/slot_diff_spec.rb
+spec/lib/strokedb/sync/spec_helper.rb
+spec/lib/strokedb/sync/store_sync_spec.rb
+spec/lib/strokedb/sync/stroke_diff/array_spec.rb
+spec/lib/strokedb/sync/stroke_diff/complex_spec.rb
+spec/lib/strokedb/sync/stroke_diff/hash_spec.rb
+spec/lib/strokedb/sync/stroke_diff/scalar_spec.rb
+spec/lib/strokedb/sync/stroke_diff/spec_helper.rb
+spec/lib/strokedb/sync/stroke_diff/string_spec.rb
+spec/lib/strokedb/util/attach_dsl_spec.rb
+spec/lib/strokedb/util/inflect_spec.rb
+spec/lib/strokedb/util/lazy_array_spec.rb
+spec/lib/strokedb/util/lazy_mapping_array_spec.rb
+spec/lib/strokedb/util/lazy_mapping_hash_spec.rb
+spec/lib/strokedb/util/spec_helper.rb
+spec/lib/strokedb/util/uuid_spec.rb
+spec/lib/strokedb/view_spec.rb
+spec/lib/strokedb/volumes/archive_volume_spec.rb
+spec/lib/strokedb/volumes/block_volume_spec.rb
+spec/lib/strokedb/volumes/distributed_pointer_spec.rb
+spec/lib/strokedb/volumes/fixed_length_skiplist_volume_spec.rb
+spec/lib/strokedb/volumes/map_volume_spec.rb
+spec/lib/strokedb/volumes/spec_helper.rb
+spec/regression/docref_spec.rb
+spec/regression/meta_spec.rb
+spec/regression/spec_helper.rb
+spec/regression/sync_spec.rb
+spec/spec.opts
+spec/spec_helper.rb
+spec/temp/storages/database-sync/config
+spec/temp/storages/database-sync/file/bd/f6/bdf675e5-8a7b-494e-97f2-f74a14ccd95d.av
+spec/temp/storages/database-sync/file/LAST
+spec/temp/storages/database-sync/file/uindex.wal
+spec/temp/storages/database-sync/inverted_list_file/INVERTED_INDEX
+spec/temp/storages/database-sync/TIMESTAMP
+spec/temp/storages/database-sync/UUID
+spec/temp/storages/inverted_list_storage/INVERTED_INDEX
+spec/temp/storages/TIMESTAMP
+spec/temp/storages/UUID
+strokedb.gemspec
+task/benchmark.task
+task/ditz.task
+task/echoe.rb
+task/rcov.task
+task/rdoc.task
+task/rspec.task
+vendor/java_inline.rb
+vendor/rbmodexcl/mrimodexcl.rb
+vendor/rbmodexcl/rbmodexcl.rb
+vendor/rbmodexcl/rbxmodexcl.rb
+vendor/rbmodexcl/spec/unextend_spec.rb
+vendor/rbmodexcl/spec/uninclude_spec.rb
+meta/MANIFEST