strokedb 0.0.2.1 → 0.0.2.2

data/lib/strokedb/store.rb

@@ -0,0 +1,149 @@
+module StrokeDB
+
+  StoreInfo = Meta.new
+
+  class Store
+    include Enumerable
+    attr_accessor :storage, :index_store, :uuid
+    attr_reader :timestamp
+
+    def initialize(options = {})
+      @options = options.stringify_keys
+      @storage = @options['storage']
+      @index_store = @options['index']
+      initialize_files
+      autosync! unless @options['noautosync']
+      raise "Missing chunk storage" unless @storage
+    end
+
+    def find(uuid, version=nil, opts = {}, &block)
+      @storage.find(uuid,version,opts.merge(:store => self),&block)
+    end
+
+    def search(*args)
+      return [] unless @index_store
+      @index_store.find(*args)
+    end
+
+    def include?(uuid,version=nil)
+      @storage.include?(uuid,version)
+    end
+    alias_method :contains?, :include?
+
+    def head_version(uuid)
+      @storage.head_version(uuid,{ :store => self })
+    end
+
+    def save!(doc)
+      next_timestamp!
+      storage.save!(doc, timestamp)
+      # 
+      if @index_store
+        if doc.previous_version
+          raw_pdoc = find(doc.uuid,doc.previous_version,:no_instantiation => true)
+          pdoc = Document.from_raw(self,raw_pdoc.freeze,:skip_callbacks => true)
+          pdoc.extend(VersionedDocument)
+          @index_store.delete(pdoc)
+        end
+        @index_store.insert(doc)
+        @index_store.save!
+      end
+    end  
+    
+    def save_as_head!(doc)
+      @storage.save_as_head!(doc,timestamp)
+    end
+    
+
+
+    def each(options = {},&block)
+      @storage.each(options.merge(:store => self),&block)
+    end
+
+    def next_timestamp!
+      @timestamp = timestamp.next
+      update_timestamp!
+      @timestamp
+    end
+
+    def uuid
+      return @uuid if @uuid
+      @uuid = Util.random_uuid
+    end
+
+
+    def document
+      find(uuid) || StoreInfo.create!(self, :uuid => uuid)
+    end
+
+    def inspect
+      "#<Store #{uuid}>"
+    end
+
+    def autosync!
+      @autosync_mutex ||= Mutex.new
+      @autosync = nil if @autosync && !@autosync.status
+      at_exit { stop_autosync! }
+      @autosync ||= Thread.new do 
+        until @stop_autosync
+          @autosync_mutex.synchronize { storage.sync_chained_storages! }
+          sleep(1)
+        end
+      end
+    end
+
+    def stop_autosync!
+      if @autosync_mutex
+        @autosync_mutex.synchronize { @stop_autosync = true; storage.sync_chained_storages! }
+      end
+    end
+
+    def path
+      @options['path']
+    end
+
+
+    private
+
+    def initialize_files
+      FileUtils.mkdir_p(path)
+      uuid_file = File.join(path,'UUID')
+      timestamp_file = File.join(path,'TIMESTAMP')
+
+      if File.exists?(uuid_file)
+        @uuid = IO.read(uuid_file)
+      else
+        File.open(uuid_file,'w') do |f|
+          f.write(uuid)
+        end
+      end
+      if File.exists?(timestamp_file)
+        @timestamp = LTS.new(IO.read(timestamp_file).to_i,uuid)
+      else
+        @timestamp = LTS.zero(uuid)
+        update_timestamp!
+      end
+    end
+
+    def update_timestamp!
+      timestamp_file = File.join(path,'TIMESTAMP')
+      File.open(timestamp_file,'w') do |f|
+        f.write(timestamp.counter)
+      end        
+    end
+
+  end
+end
+
+require 'stores/chainable_storage'
+module StrokeDB
+  class Storage
+    include ChainableStorage
+
+    attr_accessor :authoritative_source
+
+    def initialize(opts={})
+    end
+
+  end
+end

data/lib/strokedb/stores.rb

@@ -0,0 +1,6 @@
+require 'store'
+require 'stores/chainable_storage'
+require 'stores/file_storage'
+require 'stores/memory_storage'
+require 'stores/remote_store'
+require 'stores/inverted_list_file_storage'

data/lib/{stores → strokedb/stores}/chainable_storage.rb

@@ -2,7 +2,7 @@ module StrokeDB
   module ChainableStorage
     def add_chained_storage!(storage)
       @chained_storages ||= {}
-      @chained_storages[storage] = []
+      @chained_storages[storage] = [{},[]]
       class <<self
         alias :save! :save_with_chained_storages!
       end
@@ -13,7 +13,7 @@ module StrokeDB
       @chained_storages.delete(storage)
       storage.remove_chained_storage!(self) if storage.has_chained_storage?(self)
       if @chained_storages.keys.empty?
-        class <<self
+        class << self
           alias :save! :save_without_chained_storages!
         end
       end
@@ -27,28 +27,34 @@ module StrokeDB
       return unless @chained_storages.is_a?(Hash)
       @chained_storages.each_pair do |storage, savings|
         next if storage == origin
-        savings.each {|saving| storage.save!(saving, self)}
+        savings.last.each {|saving| storage.save!(saving[0], saving[1], saving[2], self)}
         storage.sync_chained_storages!(self)
-        @chained_storages[storage] = [] 
+        @chained_storages[storage] =  [{},[]]
       end
     end
-    
+
     def sync_chained_storage!(storage)
       return unless @chained_storages.is_a?(Hash)
-      (@chained_storages[storage]||[]).each do |saving|
-        storage.save!(saving, self)
+      ((@chained_storages[storage]||[]).last||[]).each do |saving|
+        storage.save!(saving[0], saving[1], saving[2], self)
       end
-      @chained_storages[storage] = []
+      @chained_storages[storage] = [{},[]]
     end
 
-    def save_without_chained_storages!(chunk, source=nil)
-      perform_save!(chunk)
+    def save_without_chained_storages!(document, timestamp, options = {},source=nil)
+      perform_save!(document, timestamp, options)
     end
 
-    def save_with_chained_storages!(chunk, source=nil)
-      perform_save!(chunk)
-      (@chained_storages||{}).each_pair do |storage, savings|
-          savings << chunk unless  storage == source || savings.include?(chunk) # TODO: here we had a bug (storage == document), spec it
+    def save_with_chained_storages!(document, timestamp, options = {},source=nil)
+      perform_save!(document, timestamp, options)
+      (@chained_storages||{}).each_pair do |storage, accumulator|
+        next if storage == source
+        hsh = [document,timestamp,options].hash
+        hashes, savings = accumulator
+        unless hashes.has_key?(hsh) # TODO: here we had a bug (storage == document), spec it
+          hashes[hsh] = true
+          savings << [document,timestamp,options]      
+        end
       end
     end
 

data/lib/strokedb/stores/file_storage.rb

@@ -0,0 +1,118 @@
+module StrokeDB
+  class FileStorage < Storage
+
+    def initialize(options = {})
+      @options = options.stringify_keys
+      initialize_files
+    end
+
+    def save_as_head!(document, timestamp)
+      write(document.uuid, document, timestamp)
+    end      
+
+    def find(uuid, version=nil, opts = {}, &block)
+      uuid_version = uuid + (version ? ".#{version}" : "")
+      key = uuid.to_raw_uuid + (version ? version.to_raw_uuid : RAW_FULL_UUID)
+      if (ptr = @uindex.find(key)) && (ptr != "\x00" * 20) # no way ptr will be zero
+        raw_doc = StrokeDB::deserialize(read_at_ptr(ptr))[0]
+        unless opts[:no_instantiation]
+          doc = Document.from_raw(opts[:store], raw_doc.freeze, &block) # FIXME: there should be a better source for store (probably)
+          doc.extend(VersionedDocument) if version
+          doc
+        else
+          raw_doc
+        end
+      end
+    end
+
+    def include?(uuid,version=nil)
+      !!find(uuid,version,:no_instantiation => true)
+    end
+    # using #include? to match with Array, but #contains sounds much nicer
+    alias_method :contains?, :include?
+
+    def head_version(uuid, opts = {})
+      if doc = find(uuid,nil,opts.merge({ :no_instantiation => true }))
+        doc['version']
+      end	 
+    end
+
+    def each(options = {})
+      after = options[:after_timestamp]
+      include_versions = options[:include_versions]
+      @container.each do |key, value|
+        next if after && (value[1] <= after)
+        if uuid_match = key.match(/^#{UUID_RE}$/) || (include_versions && uuid_match = key.match(/#{UUID_RE}./) )
+          yield Document.from_raw(options[:store],value[0])
+        end
+      end
+    end
+
+    def perform_save!(document, timestamp, options = {})
+      position = @archive.insert(StrokeDB::serialize([document,timestamp.counter]))
+      ptr = DistributedPointer.pack(@archive.raw_uuid,position)
+      uuid = document.raw_uuid
+      @uindex.insert(uuid + RAW_FULL_UUID, ptr) if options[:head] || !document.is_a?(VersionedDocument)
+      @uindex.insert(uuid + document.version.to_raw_uuid, ptr) unless options[:head]
+    rescue ArchiveVolume::VolumeCapacityExceeded	 
+      create_new_archive!
+    end
+
+    def create_new_archive!
+      @archive = ArchiveVolume.new(:path => @options['path'])
+      File.open(File.join(@options['path'],'LAST'),'w') do |f|
+        f.write(@archive.uuid)
+      end
+    end
+
+    def last_archive_uuid
+      last_filename = File.join(@options['path'],'LAST')
+      if File.exists?(last_filename)
+        IO.read(last_filename) 
+      else 
+        uuid = Util.random_uuid
+        File.open(last_filename,'w') do |f|
+          f.write(uuid)
+        end
+        uuid
+      end
+    end
+
+    def read_at_ptr(ptr)
+      dptr = DistributedPointer.unpack(ptr)
+      volume_uuid = dptr.volume_uuid
+      if volume_uuid == @archive.uuid.to_raw_uuid
+        @archive.read(dptr.offset)
+      else
+        archive = ArchiveVolume.new(:path => @options['path'], :uuid => volume_uuid)
+        result = archive.read(dptr.offset)
+        archive.close!
+        result
+      end
+    end
+    
+    def path
+      @options['path']
+    end
+
+    def clear!
+      FileUtils.rm_rf(@options['path'])
+      initialize_files
+    end
+    
+    def close!
+      @archive.close!
+      @uindex.close!
+    end
+      
+    
+    private
+    
+    def initialize_files
+      FileUtils.mkdir_p(@options['path'])
+      @archive = ArchiveVolume.new(:path => @options['path'], :uuid => last_archive_uuid)
+      @uindex = FixedLengthSkiplistVolume.new(:path => File.join(@options['path'],'uindex'), :key_length => 32 , :value_length => 20)
+    end
+
+  end
+end

data/lib/{stores/inverted_list_index → strokedb/stores}/inverted_list_file_storage.rb

@@ -53,4 +53,54 @@ module StrokeDB
       "#{@path}/INVERTED_INDEX"
     end
   end
+  
+  
+  
+  class InvertedListIndex
+    attr_accessor :storage, :document_store
+    def initialize(storage)
+      @storage = storage
+      @list = nil
+    end
+    
+    def find_uuids(*args)
+      list.find(*args)
+    end
+    
+    def find(*args)
+      find_uuids(*args).map do |uuid|
+        @document_store.find(uuid)
+      end
+    end
+    
+    def insert(doc)
+      slots = indexable_slots_for_doc(doc)
+      q = PointQuery.new(slots)
+      list.insert(q.slots, doc.uuid)
+    end
+     
+    def delete(doc)
+      slots = indexable_slots_for_doc(doc)
+      q = PointQuery.new(slots)
+      list.delete(q.slots, doc.uuid)
+    end
+    
+    def save!
+      @storage.save!(list)
+    end
+  
+  private
+    
+    def indexable_slots_for_doc(doc)
+      raw_slots = doc.to_raw
+      nkeys = doc.meta['non_indexable_slots']
+      nkeys.each{|nk| raw_slots.delete(nk) } if nkeys
+      raw_slots
+    end
+    
+    def list
+      @list ||= @storage.find_list
+    end
+    
+  end
 end

data/lib/strokedb/stores/memory_storage.rb

@@ -0,0 +1,80 @@
+module StrokeDB
+  class MemoryStorage < Storage
+    
+    def initialize(options = {})
+    	@options = options.stringify_keys
+    	clear!
+    end
+    
+    def save_as_head!(document, timestamp)
+      save!(document,timestamp, :head => true)
+		end      
+    
+    def find(uuid, version=nil, opts = {},&block)
+      uuid_version = uuid + (version ? ".#{version}" : "")
+      unless raw_doc = read(uuid_version)
+        authoritative_source.find(uuid,version,opts,&block) if authoritative_source
+      else
+        unless opts[:no_instantiation]
+          doc = Document.from_raw(opts[:store], raw_doc.freeze, &block) # FIXME: there should be a better source for store (probably)
+          doc.extend(VersionedDocument) if version
+          doc
+        else
+          raw_doc
+        end
+      end
+    end
+    
+    def include?(uuid,version=nil)
+      uuid_version = uuid + (version ? ".#{version}" : "")
+      !@container.find(uuid_version).nil?
+    end
+    alias_method :contains?, :include?
+
+    def head_version(uuid, opts = {})
+      if doc = find(uuid,nil, opts)
+      	 doc.version
+      end	 
+    end
+ 		
+ 		def each(options = {})
+      after = options[:after_timestamp]
+      include_versions = options[:include_versions]
+      @container.each do |key, value|
+          next if after && (value[1] <= after)
+          if uuid_match = key.match(/^#{UUID_RE}$/) || (include_versions && uuid_match = key.match(/#{UUID_RE}./) )
+            yield Document.from_raw(options[:store],value[0])
+          end
+      end
+ 		end
+
+    def perform_save!(document, timestamp, options = {})
+      uuid = document.uuid
+      version = document.version
+      raw_document = document.to_raw
+      uuid_version = uuid + (version ? ".#{version}" : "")
+      write(uuid, raw_document, timestamp) if options[:head] || !document.is_a?(VersionedDocument)
+      write(uuid_version, raw_document, timestamp) unless options[:head]
+    end
+
+    def clear!
+      @container = SimpleSkiplist.new
+    end
+
+    def close!
+    end
+    
+    private
+
+    def read(key)
+      if record = @container.find(key)
+      	 record.first
+      end
+    end
+        
+    def write(key, document, timestamp)
+      @container.insert(key, [document, timestamp.counter])
+    end
+    
+  end
+end