euston-eventstore 1.0.0

data/lib/euston-eventstore/optimistic_event_stream.rb

@@ -0,0 +1,106 @@
+module Euston
+  module EventStore
+    class OptimisticEventStream
+      def initialize(options)
+        @persistence = options[:persistence]
+        @committed_events = []
+        @uncommitted_events = []
+        @uncommitted_headers = {}
+        @commit_sequence = 0
+        @identifiers = []
+
+        if options.has_key? :snapshot
+          snapshot = options[:snapshot]
+          @stream_id = snapshot.stream_id
+          commits = @persistence.get_from @stream_id, snapshot.stream_revision, options[:max_revision]
+          populate_stream snapshot.stream_revision + 1, options[:max_revision], commits
+          @stream_revision = snapshot.stream_revision + committed_events.length
+        else
+          @stream_id = options[:stream_id]
+          @stream_revision = 0
+          min_revision = options[:min_revision] ||= nil
+          max_revision = options[:max_revision] ||= nil
+
+          unless min_revision.nil? || max_revision.nil?
+            commits = @persistence.get_from @stream_id, min_revision, max_revision
+            populate_stream min_revision, max_revision, commits
+
+            raise StreamNotFoundError if (min_revision > 0 && committed_events.empty?)
+          end
+        end
+      end
+
+      attr_reader :stream_id, :stream_revision, :commit_sequence, :committed_events, :uncommitted_events, :uncommitted_headers
+
+      def <<(event)
+        @uncommitted_events << event unless event.nil? || event.body.nil?
+      end
+
+      def clear_changes
+        @uncommitted_events = []
+        @uncommitted_headers = {}
+      end
+
+      def commit_changes(commit_id)
+        raise Euston::EventStore::DuplicateCommitError if @identifiers.include? commit_id
+
+        return unless has_changes
+
+        begin
+          persist_changes commit_id
+        rescue ConcurrencyError => e
+          commits = @persistence.get_from stream_id, stream_revision + 1, FIXNUM_MAX
+          populate_stream stream_revision + 1, FIXNUM_MAX, commits
+
+          raise e
+        end
+      end
+
+      protected
+
+      def copy_values_to_new_commit(commit_id)
+        Euston::EventStore::Commit.new :stream_id => stream_id,
+                                       :stream_revision => stream_revision + uncommitted_events.length,
+                                       :commit_id => commit_id,
+                                       :commit_sequence => commit_sequence + 1,
+                                       :commit_timestamp => Time.now.utc,
+                                       :headers => uncommitted_headers,
+                                       :events => uncommitted_events
+      end
+
+      def has_changes
+        !uncommitted_events.empty?
+      end
+
+      def persist_changes(commit_id)
+        commit = copy_values_to_new_commit commit_id
+        @persistence.commit commit
+
+        populate_stream stream_revision + 1, commit.stream_revision, [ commit ]
+        clear_changes
+      end
+
+      def populate_stream(min_revision, max_revision, commits = [])
+        commits.each do |commit|
+          @identifiers << commit.commit_id
+          @commit_sequence = commit.commit_sequence
+
+          current_revision = commit.stream_revision - commit.events.length + 1
+
+          return if current_revision > max_revision
+
+          commit.events.each do |event|
+            break if current_revision > max_revision
+
+            unless current_revision < min_revision
+              @committed_events << event
+              @stream_revision = current_revision
+            end
+
+            current_revision += 1
+          end
+        end
+      end
+    end
+  end
+end

data/lib/euston-eventstore/persistence/mongodb/mongo_commit.rb

@@ -0,0 +1,82 @@
+module Euston
+  module EventStore
+    module Persistence
+      module Mongodb
+        module MongoCommit
+          extend ::ActiveSupport::Concern
+
+          included do
+            alias_method :original_initialize, :initialize
+            alias_method :initialize, :mongo_initialize
+          end
+
+          class << self
+            def from_hash(hash)
+              return nil if hash.nil?
+
+              id = hash['_id']
+              events = hash['events'].sort_by { |e| e["stream_revision"] }.to_a
+              stream_revision = events.last['stream_revision']
+              events = events.map { |e| Euston::EventStore::Persistence::Mongodb::MongoEventMessage.from_hash e['payload'] }
+
+              Euston::EventStore::Commit.new :stream_id => id['stream_id'],
+                                             :stream_revision => stream_revision,
+                                             :commit_id => hash['commit_id'],
+                                             :commit_sequence => id['commit_sequence'],
+                                             :commit_timestamp => hash['commit_timestamp'],
+                                             :headers => hash['headers'].recursive_symbolize_keys!,
+                                             :events => events
+            end
+          end
+
+          def mongo_initialize(hash)
+            original_initialize(hash)
+            @dispatched = hash[:dispatched]
+          end
+
+          attr_reader :dispatched
+
+          def to_hash
+            {
+              :_id => { :stream_id => stream_id, :commit_sequence => commit_sequence },
+              :commit_id => commit_id,
+              :commit_timestamp => commit_timestamp.to_f,
+              :dispatched => dispatched || false,
+              :events => events.map { |e| e.to_hash },
+              :headers => headers
+            }
+          end
+
+          def to_mongo_commit
+            mongo_stream_revision = stream_revision - (events.length - 1)
+            mongo_events = events.map do |e|
+              hash = { :stream_revision => mongo_stream_revision, :payload => e.to_hash }
+              mongo_stream_revision += 1
+              hash
+            end
+
+            {
+              :_id => { :stream_id => stream_id, :commit_sequence => commit_sequence },
+              :commit_id => commit_id,
+              :commit_timestamp => commit_timestamp.to_f,
+              :headers => headers,
+              :events => mongo_events,
+              :dispatched => false
+            }
+          end
+
+          def to_id_query
+            {
+              '_id.commit_sequence' => commit_sequence,
+              '_id.stream_id' => stream_id
+            }
+          end
+        end
+      end
+    end
+
+    class Commit
+      include Persistence::Mongodb::MongoCommit
+    end
+  end
+end

data/lib/euston-eventstore/persistence/mongodb/mongo_commit_id.rb

@@ -0,0 +1,16 @@
+module Euston
+  module EventStore
+    module Persistence
+      module Mongodb
+        module MongoCommitId
+          def initialize(stream_id, commit_sequence)
+            @stream_id = stream_id
+            @commit_sequence = commit_sequence
+          end
+
+          attr_reader :stream_id, :commit_sequence
+        end
+      end
+    end
+  end
+end

data/lib/euston-eventstore/persistence/mongodb/mongo_config.rb

@@ -0,0 +1,28 @@
+require 'singleton'
+
+module Euston
+  module EventStore
+    module Persistence
+      module Mongodb
+        class Config
+          include ::Singleton
+
+          def host
+            @host ||= 'localhost'
+          end
+
+          def port
+            @port ||= 27017
+          end
+
+          def options
+            @options ||= { :safe => { :fsync => true }}
+          end
+
+          attr_writer :host, :port, :options
+          attr_accessor :database, :logger
+        end
+      end
+    end
+  end
+end

data/lib/euston-eventstore/persistence/mongodb/mongo_event_message.rb

@@ -0,0 +1,31 @@
+module Euston
+  module EventStore
+    module Persistence
+      module Mongodb
+        module MongoEventMessage
+          extend ::ActiveSupport::Concern
+
+          class << self
+            def from_hash(hash)
+              {}.recursive_symbolize_keys!
+              message = EventMessage.new hash['body'].recursive_symbolize_keys!
+              message.instance_variable_set :@headers, hash['headers'].recursive_symbolize_keys!
+              message
+            end
+          end
+
+          def to_hash
+            {
+              :headers => headers,
+              :body => body.to_hash.recursive_stringify_symbol_values!
+            }
+          end
+        end
+      end
+    end
+
+    class EventMessage
+      include Persistence::Mongodb::MongoEventMessage
+    end
+  end
+end

data/lib/euston-eventstore/persistence/mongodb/mongo_persistence_engine.rb

@@ -0,0 +1,167 @@
+module Euston
+  module EventStore
+    module Persistence
+      module Mongodb
+        class MongoPersistenceEngine
+          def initialize(store)
+            @store = store
+
+            collection_names = store.collection_names
+            store.create_collection 'commits'  unless collection_names.include? 'commits'    # :safe = true
+            store.create_collection 'snapshot' unless collection_names.include? 'snapshot'   # :safe = false
+            store.create_collection 'streams'  unless collection_names.include? 'streams'    # :safe = false
+          end
+
+          def add_snapshot(snapshot)
+            return false if snapshot.nil?
+
+            begin
+              mongo_snapshot = snapshot.is_a?(Hash) ? snapshot : snapshot.to_hash
+              id = { '_id' => mongo_snapshot[:_id] }
+
+              persisted_snapshots.update(id, { 'payload' => mongo_snapshot[:payload] }.merge(id), { :upsert => true })
+
+              # jmongo's find_one is broken
+              if defined?(JMongo)
+                stream_head = MongoStreamHead.from_hash persisted_stream_heads.find({ '_id' => snapshot.stream_id }).to_a.first
+              else
+                stream_head = MongoStreamHead.from_hash persisted_stream_heads.find_one({ '_id' => snapshot.stream_id })
+              end
+
+              unsnapshotted = stream_head.head_revision - snapshot.stream_revision
+              persisted_stream_heads.update({ '_id' => snapshot.stream_id },
+                                            { '$set' => { 'snapshot_revision' => snapshot.stream_revision, 'unsnapshotted' => unsnapshotted } })
+              return true
+            rescue Mongo::OperationFailure
+              return false
+            end
+          end
+
+          def commit(attempt)
+            try_mongo do
+              commit = attempt.to_mongo_commit
+
+              begin
+                # for concurrency / duplicate commit detection safe mode is required
+                persisted_commits.insert commit, :safe => true
+                update_stream_head_async attempt.stream_id, attempt.stream_revision, attempt.events.length
+              rescue Mongo::OperationFailure, NativeException => e
+                raise(Euston::EventStore::StorageError, e.message, e.backtrace) unless e.message.include? CONCURRENCY_EXCEPTION
+
+                # jmongo's find_one is broken
+                if defined?(JMongo)
+                  committed = persisted_commits.find(attempt.to_id_query).to_a.first
+                else
+                  committed = persisted_commits.find_one(attempt.to_id_query)
+                end
+
+                raise Euston::EventStore::DuplicateCommitError if !committed.nil? && committed['commit_id'] == attempt.commit_id
+                raise Euston::EventStore::ConcurrencyError
+              end
+            end
+          end
+
+          def get_from(options)
+            try_mongo do
+              if options.has_key? :timestamp
+                query = { 'commit_timestamp' => { '$gte' => options[:timestamp].to_f } }
+                order = [ 'commit_timestamp', Mongo::ASCENDING ]
+              else
+                query = { '_id.stream_id' => options[:stream_id],
+                          'events.stream_revision' => { '$gte' => options[:min_revision], '$lte' => options[:max_revision] } }
+
+                order = [ 'events.stream_revision', Mongo::ASCENDING ]
+              end
+
+              persisted_commits.find(query).sort(order).to_a.map { |hash| MongoCommit.from_hash hash }
+            end
+          end
+
+          def get_snapshot(stream_id, max_revision)
+            try_mongo do
+              query = { '_id' => {  '$gt' => { 'stream_id' => stream_id, 'stream_revision' => nil },
+                                   '$lte' => { 'stream_id' => stream_id, 'stream_revision' => max_revision } } }
+              order = [ '_id', Mongo::DESCENDING ]
+
+              persisted_snapshots.find(query).sort(order).limit(1).to_a.map { |hash| MongoSnapshot::from_hash hash }.first
+            end
+          end
+
+          def get_streams_to_snapshot(max_threshold)
+            try_mongo do
+              query = { 'unsnapshotted' => { '$gte' => max_threshold } }
+              order = [ 'unsnapshotted', Mongo::DESCENDING ]
+
+              persisted_stream_heads.find(query).sort(order).to_a.map { |hash| MongoStreamHead.from_hash hash }
+            end
+          end
+
+          def get_undispatched_commits
+            try_mongo do
+              query = { 'dispatched' => false }
+              order = [ 'commit_timestamp', Mongo::ASCENDING ]
+
+              persisted_commits.find(query).sort(order).to_a.map { |hash| MongoCommit.from_hash hash }
+            end
+          end
+
+          def init
+            try_mongo do
+              persisted_commits.ensure_index [ ['dispatched', Mongo::ASCENDING],
+                                               ['commit_timestamp', Mongo::ASCENDING] ], :unique => false, :name => 'dispatched_index'
+
+              persisted_commits.ensure_index [ ['_id.stream_id', Mongo::ASCENDING],
+                                               ['events.stream_revision', Mongo::ASCENDING] ], :unique => true,  :name => 'get_from_index'
+
+              persisted_commits.ensure_index [ ['commit_timestamp', Mongo::ASCENDING] ], :unique => false, :name => 'commit_timestamp_index'
+
+              persisted_stream_heads.ensure_index [ ['unsnapshotted', Mongo::ASCENDING] ], :unique => false, :name => 'unsnapshotted_index'
+            end
+            self
+          end
+
+          def mark_commit_as_dispatched(commit)
+            try_mongo do
+              persisted_commits.update commit.to_id_query, { '$set' => { 'dispatched' => true }}
+            end
+          end
+
+          private
+
+          def persisted_commits
+            @store.collection 'commits'
+          end
+
+          def persisted_snapshots
+            @store.collection 'snapshots'
+          end
+
+          def persisted_stream_heads
+            @store.collection 'streams'
+          end
+
+          def try_mongo(&block)
+            begin
+              yield block
+            rescue Mongo::ConnectionError => e
+              raise Euston::EventStore::StorageUnavailableError, e.to_s, e.backtrace
+            rescue Mongo::MongoDBError => e
+              raise Euston::EventStore::StorageError, e.to_s, e.backtrace
+            end
+          end
+
+          def update_stream_head_async(stream_id, stream_revision, events_count)
+            Thread.fork do
+              persisted_stream_heads.update(
+                { '_id' => stream_id },
+                { '$set' => { 'head_revision' => stream_revision }, '$inc' => { 'snapshot_revision' => 0, 'unsnapshotted' => events_count } },
+                { :upsert  => true })
+            end
+          end
+
+          CONCURRENCY_EXCEPTION = "E1100"
+        end
+      end
+    end
+  end
+end

data/lib/euston-eventstore/persistence/mongodb/mongo_persistence_factory.rb

@@ -0,0 +1,31 @@
+if RUBY_PLATFORM.to_s == 'java'
+  module JMongo
+    module BasicDBObjectExtentions
+      include HashKeys
+    end
+  end
+
+  require 'jmongo'
+else
+  require 'mongo'
+end
+
+module Euston
+  module EventStore
+    module Persistence
+      module Mongodb
+        class MongoPersistenceFactory
+          def self.build
+            config = Config.instance
+            connection = ::Mongo::Connection.new(config.host, config.port, config.options)
+
+            MongoPersistenceEngine.new connection.db(config.database)
+          end
+          def self.build_with_proxy()
+            ZmqPersistenceEngineProxy.new(build.init)
+          end
+        end
+      end
+    end
+  end
+end