strokedb 0.0.2

data/lib/stores/skiplist_store/chunk_storage.rb

@@ -0,0 +1,21 @@
+module StrokeDB
+  class ChunkStorage
+    include ChainableStorage
+
+    attr_accessor :authoritative_source
+
+    def initialize(opts={})
+    end
+
+    def find(uuid)
+      unless result = read(chunk_path(uuid)) 
+        if authoritative_source
+          result = authoritative_source.find(uuid) 
+          save!(result, authoritative_source) if result
+        end
+      end
+      result
+    end
+
+  end
+end

data/lib/stores/skiplist_store/file_chunk_storage.rb

@@ -0,0 +1,44 @@
+module StrokeDB
+  class FileChunkStorage < ChunkStorage
+    attr_accessor :path
+
+    def initialize(opts={})
+      opts = opts.stringify_keys 
+      @path = opts['path']
+    end
+    
+    def delete!(chunk_uuid)
+      FileUtils.rm_rf(chunk_path(chunk_uuid))
+    end
+
+    def clear!
+      FileUtils.rm_rf @path
+    end
+    
+  private
+    
+    def perform_save!(chunk)
+      FileUtils.mkdir_p @path
+      write(chunk_path(chunk.uuid), chunk)
+    end
+    
+    def read(path)
+      return nil unless File.exist?(path)
+      raw_chunk = StrokeDB.deserialize(IO.read(path))
+      Chunk.from_raw(raw_chunk) do |chunk|
+        chunk.next_chunk = find(chunk.next_chunk_uuid) if chunk.next_chunk_uuid
+      end
+    end
+    
+    def write(path, chunk)
+      File.open path, "w+" do |f|
+        f.write StrokeDB.serialize(chunk.to_raw)
+      end
+    end
+    
+    def chunk_path(uuid)
+      "#{@path}/#{uuid}"
+    end
+    
+  end
+end

data/lib/stores/skiplist_store/memory_chunk_storage.rb

@@ -0,0 +1,37 @@
+module StrokeDB
+  class MemoryChunkStorage < ChunkStorage
+    attr_accessor :chunks_cache
+
+    def initialize(opts={})
+      @chunks_cache = {}
+    end
+    
+    def delete!(chunk_uuid)
+      write(chunk_path(chunk_uuid), nil)
+    end
+    
+    def clear!
+      @chunks_cache.clear
+    end
+  
+  private
+
+    def perform_save!(chunk)
+      write(chunk_path(chunk.uuid), chunk)
+    end
+    
+    def read(path)
+      @chunks_cache[path]
+    end
+    
+    def write(path, chunk)
+      @chunks_cache[path] = chunk
+    end
+    
+    def chunk_path(uuid)
+      uuid
+    end
+    
+    
+  end
+end

data/lib/stores/skiplist_store/skiplist_store.rb

@@ -0,0 +1,217 @@
+module StrokeDB
+  class SkiplistStore < Store
+    include Enumerable
+    attr_accessor :chunk_storage, :cut_level, :index_store
+
+    def initialize(opts={})
+      opts = opts.stringify_keys
+      @chunk_storage = opts['storage']
+      @cut_level = opts['cut_level'] || 4
+      @index_store = opts['index']
+      autosync! unless opts['noautosync']
+      raise "Missing chunk storage" unless @chunk_storage
+    end
+
+    def find(uuid, version=nil, opts = {})
+      uuid_version = uuid + (version ? ".#{version}" : "")
+      master_chunk = @chunk_storage.find('MASTER')
+      return nil unless master_chunk  # no master chunk yet
+      chunk_uuid = master_chunk.find_nearest(uuid_version, nil)
+      return nil unless chunk_uuid # no chunks in master chunk yet
+      chunk = @chunk_storage.find(chunk_uuid)
+      return nil unless chunk
+
+      raw_doc = chunk.find(uuid_version)
+
+      if raw_doc
+        return raw_doc if opts[:no_instantiation]
+        doc = Document.from_raw(self, raw_doc.freeze)
+        doc.extend(VersionedDocument) if version
+        return doc
+      end
+      nil
+    end
+
+    def search(*args)
+      return [] unless @index_store
+      @index_store.find(*args)
+    end
+
+    def exists?(uuid, version=nil)
+      !!find(uuid, version, :no_instantiation => true)
+    end
+
+    def head_version(uuid)
+      raw_doc = find(uuid, nil, :no_instantiation => true)
+      return raw_doc['version'] if raw_doc
+      nil
+    end
+
+    def save!(doc)
+      master_chunk = find_or_create_master_chunk
+      next_timestamp
+
+      insert_with_cut(doc.uuid, doc, master_chunk) unless doc.is_a?(VersionedDocument)
+      insert_with_cut("#{doc.uuid}.#{doc.version}", doc, master_chunk)
+
+      update_master_chunk!(doc, master_chunk)
+    end  
+    
+    def save_as_head!(doc)
+      master_chunk = find_or_create_master_chunk
+      insert_with_cut(doc.uuid, doc, master_chunk)
+      update_master_chunk!(doc, master_chunk)
+    end
+    
+
+    def full_dump
+      puts "Full storage dump:"
+      m = @chunk_storage.find('MASTER')
+      puts "No master!" unless m
+      m.each do |node|
+        puts "[chunk: #{node.key}]"
+        chunk = @chunk_storage.find(node.value)
+        if chunk
+          chunk.each do |node|
+            puts "    [doc: #{node.key}] => {uuid: #{node.value['__uuid__']}, version: #{node.value['version']}, previous_version: #{node.value['previous_version']}"
+          end
+        else
+          puts "    nil! (but in MASTER somehow?...)"
+        end
+      end
+    end
+
+    def each(options = {})
+      return nil unless m = @chunk_storage.find('MASTER')  # no master chunk yet
+      after = options[:after_timestamp]
+      include_versions = options[:include_versions]
+      m.each do |node|
+        chunk = @chunk_storage.find(node.value)
+        next unless chunk
+        next if after && chunk.timestamp <= after
+
+        chunk.each do |node| 
+          next if after && (node.timestamp <= after)
+          if uuid_match = node.key.match(/^#{UUID_RE}$/) || (include_versions && uuid_match = node.key.match(/#{UUID_RE}./) )
+            yield Document.from_raw(self, node.value) 
+          end
+        end
+      end
+    end
+
+    def timestamp
+      @timestamp ||= (lts = find_or_create_master_chunk.timestamp) ? LTS.from_raw(lts) : LTS.zero(uuid)
+    end
+    def next_timestamp
+      @timestamp = timestamp.next
+    end
+
+    def uuid
+      return @uuid if @uuid
+      master_chunk = @chunk_storage.find('MASTER')
+      unless master_chunk
+        @uuid = Util.random_uuid
+      else
+        @uuid = master_chunk.store_uuid
+      end
+      @uuid
+    end
+
+    def document
+      find(uuid) || StoreInfo.create!(self,:kind => 'skiplist', :uuid => uuid)
+    end
+
+    def empty?
+      !@chunk_storage.find('MASTER')
+    end
+
+    def inspect
+      "#<Skiplist store #{uuid}#{empty? ? " (empty)" : ""}>"
+    end
+
+    def autosync!
+      @autosync_mutex ||= Mutex.new
+      @autosync = nil if @autosync && !@autosync.status
+      at_exit { stop_autosync! }
+      @autosync ||= Thread.new do 
+        until @stop_autosync
+          @autosync_mutex.synchronize { chunk_storage.sync_chained_storages! }
+          sleep(1)
+        end
+      end
+    end
+    
+    def stop_autosync!
+      if @autosync_mutex
+        @autosync_mutex.synchronize { @stop_autosync = true; chunk_storage.sync_chained_storages! }
+      end
+    end
+    
+
+    private
+
+    def insert_with_cut(uuid, doc, master_chunk)
+      chunk_uuid = master_chunk.find_nearest(uuid)
+      unless chunk_uuid && chunk = @chunk_storage.find(chunk_uuid)
+        chunk = Chunk.new(@cut_level)
+      end
+      a, b = chunk.insert(uuid, doc.to_raw, nil, timestamp.counter)
+      [a,b].compact.each do |chunk|
+        chunk.store_uuid = self.uuid
+        chunk.timestamp = timestamp.counter
+      end
+      # if split
+      if b
+        # rename chunk if the first chunk inconsistency detected 
+        if a.uuid != a.first_uuid
+          old_uuid = a.uuid
+          a.uuid = a.first_uuid
+          @chunk_storage.save!(a)
+          master_chunk.insert(a.uuid, a.uuid)
+          # remove old chunk
+          @chunk_storage.delete!(old_uuid)
+          master_chunk.delete(old_uuid)
+        else
+          @chunk_storage.save!(a)
+          master_chunk.insert(a.uuid, a.uuid)
+        end
+        @chunk_storage.save!(b)
+        master_chunk.insert(b.uuid, b.uuid)
+      else
+        @chunk_storage.save!(a)
+        master_chunk.insert(a.uuid, a.uuid)
+      end
+    end
+
+    def find_or_create_master_chunk
+      if master_chunk = @chunk_storage.find('MASTER')
+        return master_chunk 
+      end
+      master_chunk = Chunk.new(999)
+      master_chunk.uuid = 'MASTER'
+      master_chunk.store_uuid = uuid
+      @chunk_storage.save!(master_chunk)
+      master_chunk
+    end
+    
+    
+    def update_master_chunk!(doc, master_chunk)
+      @chunk_storage.save!(master_chunk)
+
+      # Update index
+      if @index_store
+        if doc.previous_version
+          raw_pdoc = find(doc.uuid, doc.previous_version, :no_instantiation => true)
+          pdoc = Document.from_raw(self, raw_pdoc.freeze, :skip_callbacks => true)
+          pdoc.extend(VersionedDocument)
+          @index_store.delete(pdoc)
+        end
+        @index_store.insert(doc)
+        @index_store.save!
+      end
+    end
+
+
+
+  end
+end

data/lib/stores/store.rb

@@ -0,0 +1,5 @@
+module StrokeDB
+  class Store
+  end
+  StoreInfo = Meta.new(:uuid => STORE_INFO_UUID)
+end

data/lib/sync/chain_sync.rb

@@ -0,0 +1,38 @@
+module StrokeDB
+  # You may mix-in this into specific sync implementations
+  module ChainSync
+    # We have 2 chains as an input: our chain ("to") and 
+    # a foreign chain ("from"). We're going to calculate
+    # the difference between those chains to know how to
+    # implement synchronization.
+    #
+    # There're two cases:
+    # 1) from is a subset of to -> nothing to sync
+    # 2) to is a subset of from -> fast-forward merge
+    # 3) else: merge case: return base, head_from & head_to
+    def sync_chains(from, to)
+      common = from & to
+      raise NonMatchingChains, "no common element found" if common.empty?
+      base = common[common.size - 1]
+      ifrom = from.index(base)
+      ito   = to.index(base)
+      
+      # from:  -------------base
+      # to:    -----base----head
+      if ifrom == from.size - 1
+        :up_to_date
+      
+      # from:  -----base----head
+      # to:    -------------base
+      elsif ito == to.size - 1
+        [ :fast_forward, from[ifrom..-1] ]
+      
+      # from:  -----base--------head
+      # to:    --------base-----head
+      else
+        [ :merge, from[ifrom..-1], to[ito..-1] ]
+      end
+    end
+    class NonMatchingChains < Exception; end
+  end
+end

data/lib/sync/diff.rb

@@ -0,0 +1,126 @@
+require 'diff/lcs'
+module StrokeDB
+
+  class SlotDiffStrategy 
+    def self.diff(from, to)
+      to
+    end
+  end
+
+  class DefaultSlotDiff < SlotDiffStrategy
+    def self.diff(from, to)
+      unless from.class == to.class # if value types are not the same
+        to # then return new value
+      else
+        case to
+        when /@##{UUID_RE}/, /@##{UUID_RE}.#{VERSION_RE}/
+          to
+        when Array, String
+          ::Diff::LCS.diff(from, to).map do |d|
+            d.map do |change|  
+              change.to_a 
+            end
+          end
+        when Hash
+          ::Diff::LCS.diff(from.sort_by{|e| e.to_s}, to.sort_by{|e| e.to_s}).map do |d|
+            d.map do |change|
+              [change.to_a.first, {change.to_a.last.first => change.to_a.last.last}]
+            end
+          end
+        else
+          to 
+        end
+      end
+    end
+
+    def self.patch(from, patch)
+      case from
+      when /@##{UUID_RE}/, /@##{UUID_RE}.#{VERSION_RE}/
+        patch
+      when String, Array
+        lcs_patch = patch.map do |d|
+          d.map do |change|
+            ::Diff::LCS::Change.from_a(change)
+          end
+        end
+        ::Diff::LCS.patch!(from, lcs_patch)
+      when Hash
+        lcs_patch = patch.map do |d|
+          d.map_with_index do |change, index|
+            ::Diff::LCS::Change.from_a([change.first, index, [change.last.keys.first, change.last.values.first]])
+          end
+        end
+        diff = ::Diff::LCS.patch!(from.sort_by{|e| e.to_s}, lcs_patch)
+        hash = {}
+        diff.each do |v|
+          hash[v.first] = v.last
+        end
+        hash
+      else
+        patch
+      end
+    end
+  end
+
+  Diff = Meta.new(:uuid => DIFF_UUID) do
+
+    on_initialization do |diff|
+      diff.added_slots = {} unless diff[:added_slots]
+      diff.removed_slots = {} unless diff[:removed_slots]
+      diff.updated_slots = {} unless diff[:updated_slots] 
+      diff.send!(:compute_diff) if diff.new?
+    end
+
+    def different?
+      !updated_slots.empty? || !removed_slots.empty? || !added_slots.empty?
+    end
+
+    def patch!(document)
+      added_slots.each_pair do |addition, value|
+        document[addition] = value
+      end
+      removed_slots.keys.each do |removal|
+        document.remove_slot!(removal)
+      end
+      updated_slots.each_pair do |update, value|
+        if sk = strategy_class_for(update)
+          document[update] = sk.patch(document[update], value)
+        else
+          document[update] =value
+        end
+      end
+    end
+
+
+    protected
+
+    def compute_diff
+      additions = to.slotnames - from.slotnames 
+      additions.each do |addition|
+        self.added_slots[addition] = to[addition]
+      end
+      removals = from.slotnames - to.slotnames
+      removals.each do |removal|
+        self.removed_slots[removal] = from[removal]
+      end
+      updates = (to.slotnames - additions - ['version']).select {|slotname| to[slotname] != from[slotname]}
+      updates.each do |update|
+        unless sk = strategy_class_for(update)
+          self.updated_slots[update] = to[update]
+        else
+          self.updated_slots[update] = sk.diff(from[update], to[update]) 
+        end
+      end
+    end
+
+    def strategy_class_for(slotname)
+      if from.meta && strategy = from.meta["diff_strategy_#{slotname}"]
+        _strategy_class = strategy.camelize.constantize rescue nil
+        return _strategy_class if _strategy_class && _strategy_class.ancestors.include?(SlotDiffStrategy)
+      end
+      false
+    end
+
+  end
+  
+end

data/lib/sync/lamport_timestamp.rb

@@ -0,0 +1,81 @@
+module StrokeDB
+  class LamportTimestamp
+    MAX_COUNTER = 2**64
+    BASE        = 16
+    BASE_LENGTH = 16
+
+    attr_reader :counter, :uuid
+
+    def initialize(c = 0, __uuid = Util.random_uuid)
+      if c > MAX_COUNTER
+        raise CounterOverflow.new, "Max counter value is 2**64"
+      end
+      @counter = c
+      @uuid    = __uuid 
+    end
+    def next
+      LamportTimestamp.new(@counter + 1, @uuid)
+    end
+    def next!
+      @counter += 1
+      self
+    end
+    def dup
+      LamportTimestamp.new(@counter, @uuid)
+    end
+    def marshal_dump
+      @counter.to_s(BASE).rjust(BASE_LENGTH, '0') + @uuid
+    end 
+    def marshal_load(dumped)
+      @counter = dumped[0, BASE_LENGTH].to_i(BASE)
+      @uuid    = dumped[BASE_LENGTH, 36]
+      self
+    end
+    
+    def to_json
+      marshal_dump.to_json
+    end
+
+    # Raw format
+    def self.from_raw(raw_string)
+      new.marshal_load(raw_string)
+    end
+    def to_raw
+      marshal_dump
+    end
+
+    def to_s
+      marshal_dump
+    end
+    def <=>(other)
+      primary = (@counter <=> other.counter)
+      primary == 0 ? (@uuid <=> other.uuid) : primary
+    end
+    def ==(other)
+      @counter == other.counter && @uuid == other.uuid
+    end
+    def <(other)
+      (self <=> other) < 0
+    end
+    def <=(other)
+      (self <=> other) <= 0
+    end
+    def >(other)
+      (self <=> other) > 0
+    end
+    def >=(other)
+      (self <=> other) >= 0
+    end
+    def self.zero(__uuid = Util.random_uuid)
+      ts = new(0)
+      ts.instance_variable_set(:@uuid, __uuid)
+      ts
+    end
+    def self.zero_string
+      "0"*BASE_LENGTH + NIL_UUID
+    end
+    class CounterOverflow < Exception; end
+  end
+  LTS = LamportTimestamp
+end
+

data/lib/sync/store_sync.rb

@@ -0,0 +1,79 @@
+module StrokeDB
+
+  SynchronizationReport = Meta.new(:uuid => SYNCHRONIZATION_REPORT_UUID) do
+    on_new_document do |report|
+      report.conflicts = []
+      report.added_documents = []
+      report.fast_forwarded_documents = []
+      report.non_matching_documents = []
+    end
+  end
+  
+  SynchronizationConflict = Meta.new(:uuid => SYNCHRONIZATION_CONFLICT_UUID) do
+    def resolve!
+      # by default, do nothing
+    end
+  end
+  
+  class Store
+    def sync!(docs, _timestamp=nil)
+      _timestamp_counter = timestamp.counter
+      report = SynchronizationReport.new(self, :store_document => document, :timestamp => _timestamp_counter)
+      existing_chain = {}
+      docs.group_by {|doc| doc.uuid}.each_pair do |uuid, versions|
+        doc = find(uuid)
+        existing_chain[uuid] = doc.versions.all_versions if doc 
+      end
+      case _timestamp
+      when Numeric
+        @timestamp = LTS.new(_timestamp, timestamp.uuid) 
+      when LamportTimestamp
+        @timestamp = LTS.new(_timestamp.counter, timestamp.uuid)
+      else
+      end
+      docs.each {|doc| save!(doc) unless exists?(doc.uuid, doc.version)}
+      docs.group_by {|doc| doc.uuid}.each_pair do |uuid, versions|
+        incoming_chain = find(uuid, versions.last.version).versions.all_versions
+        if existing_chain[uuid].nil? or existing_chain[uuid].empty? # It is a new document
+          added_doc = find(uuid, versions.last.version)
+          save_as_head!(added_doc)
+          report.added_documents << added_doc
+        else
+          begin
+            sync = sync_chains(incoming_chain.reverse, existing_chain[uuid].reverse)
+          rescue NonMatchingChains
+            # raise NonMatchingDocumentCondition.new(uuid) # that will definitely leave garbage in the store (FIXME?)
+            non_matching_doc = find(uuid)
+            report.non_matching_documents << non_matching_doc
+            next
+          end
+          resolution = sync.is_a?(Array) ? sync.first : sync
+          case resolution
+          when :up_to_date
+            # nothing to do
+          when :merge
+            report.conflicts << SynchronizationConflict.create!(self, :document => find(uuid), :rev1 => sync[1], :rev2 => sync[2])
+          when :fast_forward
+            fast_forwarded_doc = find(uuid, sync[1].last)
+            save_as_head!(fast_forwarded_doc)
+            report.fast_forwarded_documents << fast_forwarded_doc
+          else
+            raise "Invalid sync resolution #{resolution}"
+          end
+        end
+      end
+      report.conflicts.each do |conflict|
+        if resolution_strategy = conflict.document.meta[:resolution_strategy]
+          conflict.metas << resolution_strategy
+          conflict.save!
+        end
+        conflict.resolve!
+      end
+      report.save!
+    end
+    private
+    
+    include ChainSync
+    
+  end
+end