event_sourcery-postgres 0.3.0

data/lib/event_sourcery/postgres/optimised_event_poll_waiter.rb

@@ -0,0 +1,81 @@
+module EventSourcery
+  module Postgres
+    # Optimise poll interval with Postgres listen/notify
+    class OptimisedEventPollWaiter
+      ListenThreadDied = Class.new(StandardError)
+
+      def initialize(pg_connection:, timeout: 30, after_listen: proc { })
+        @pg_connection = pg_connection
+        @timeout = timeout
+        @events_queue = QueueWithIntervalCallback.new
+        @after_listen = after_listen
+      end
+
+      def poll(after_listen: proc { }, &block)
+        @events_queue.callback = proc do
+          ensure_listen_thread_alive!
+          block.call
+        end
+        start_async(after_listen: after_listen)
+        catch(:stop) {
+          block.call
+          loop do
+            ensure_listen_thread_alive!
+            wait_for_new_event_to_appear
+            clear_new_event_queue
+            block.call
+          end
+        }
+      ensure
+        shutdown!
+      end
+
+      private
+
+      def shutdown!
+        if @listen_thread.alive?
+          @listen_thread.kill
+        end
+      end
+
+      def ensure_listen_thread_alive!
+        if !@listen_thread.alive?
+          raise ListenThreadDied
+        end
+      end
+
+      def wait_for_new_event_to_appear
+        @events_queue.pop
+      end
+
+      def clear_new_event_queue
+        @events_queue.clear
+      end
+
+      def start_async(after_listen: nil)
+        after_listen_callback = if after_listen
+                                  proc {
+                                    after_listen.call
+                                    @after_listen.call if @after_listen
+                                  }
+                                else
+                                  @after_listen
+                                end
+        @listen_thread = Thread.new { listen_for_new_events(loop: true,
+                                                            after_listen: after_listen_callback,
+                                                            timeout: @timeout) }
+      end
+
+      def listen_for_new_events(loop: true, after_listen: nil, timeout: 30)
+        @pg_connection.listen('new_event',
+                              loop: loop,
+                              after_listen: after_listen,
+                              timeout: timeout) do |_channel, _pid, _payload|
+          if @events_queue.empty?
+            @events_queue.push(:new_event_arrived)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/event_sourcery/postgres/projector.rb

@@ -0,0 +1,42 @@
+module EventSourcery
+  module Postgres
+    module Projector
+      def self.included(base)
+        base.include(EventProcessing::EventStreamProcessor)
+        base.prepend(TableOwner)
+        base.include(InstanceMethods)
+        base.class_eval do
+          alias project process
+
+          class << self
+            alias project process
+            alias projects_events processes_events
+            alias projector_name processor_name
+          end
+        end
+      end
+
+      module InstanceMethods
+        def initialize(tracker: EventSourcery::Postgres.config.event_tracker,
+                       db_connection: EventSourcery::Postgres.config.projections_database)
+          @tracker = tracker
+          @db_connection = db_connection
+        end
+
+        private
+
+        def process_events(events, subscription_master)
+          events.each do |event|
+            subscription_master.shutdown_if_requested
+            db_connection.transaction do
+              process(event)
+              tracker.processed_event(processor_name, event.id)
+            end
+            EventSourcery.logger.debug { "[#{processor_name}] Processed event: #{event.inspect}" }
+            EventSourcery.logger.info { "[#{processor_name}] Processed up to event id: #{events.last.id}" }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/event_sourcery/postgres/queue_with_interval_callback.rb

@@ -0,0 +1,33 @@
+module EventSourcery
+  module Postgres
+    class QueueWithIntervalCallback < ::Queue
+      attr_accessor :callback
+
+      def initialize(callback: proc { }, callback_interval: EventSourcery::Postgres.config.callback_interval_if_no_new_events, poll_interval: 0.1)
+        @callback = callback
+        @callback_interval = callback_interval
+        @poll_interval = poll_interval
+        super()
+      end
+
+      def pop(non_block_without_callback = false)
+        return super if non_block_without_callback
+        pop_with_interval_callback
+      end
+
+      private
+
+      def pop_with_interval_callback
+        time = Time.now
+        loop do
+          return pop(true) if !empty?
+          if @callback_interval && Time.now > time + @callback_interval
+            @callback.call
+            time = Time.now
+          end
+          sleep @poll_interval
+        end
+      end
+    end
+  end
+end

data/lib/event_sourcery/postgres/reactor.rb

@@ -0,0 +1,72 @@
+module EventSourcery
+  module Postgres
+    module Reactor
+      UndeclaredEventEmissionError = Class.new(StandardError)
+
+      def self.included(base)
+        base.include(EventProcessing::EventStreamProcessor)
+        base.extend(ClassMethods)
+        base.prepend(TableOwner)
+        base.include(InstanceMethods)
+      end
+
+      module ClassMethods
+        def emits_events(*event_types)
+          @emits_event_types = event_types
+        end
+
+        def emit_events
+          @emits_event_types ||= []
+        end
+
+        def emits_events?
+          !emit_events.empty?
+        end
+
+        def emits_event?(event_type)
+          emit_events.include?(event_type)
+        end
+      end
+
+      module InstanceMethods
+        def initialize(tracker: EventSourcery::Postgres.config.event_tracker,
+                       db_connection: EventSourcery::Postgres.config.projections_database,
+                       event_source: EventSourcery::Postgres.config.event_source,
+                       event_sink: EventSourcery::Postgres.config.event_sink)
+          @tracker = tracker
+          @event_source = event_source
+          @event_sink = event_sink
+          @db_connection = db_connection
+          if self.class.emits_events?
+            if event_sink.nil? || event_source.nil?
+              raise ArgumentError, 'An event sink and source is required for processors that emit events'
+            end
+          end
+        end
+      end
+
+      DRIVEN_BY_EVENT_PAYLOAD_KEY = :_driven_by_event_id
+
+      private
+
+      attr_reader :event_sink, :event_source
+
+      def emit_event(event_or_hash, &block)
+        event = if Event === event_or_hash
+          event_or_hash
+        else
+          Event.new(event_or_hash)
+        end
+        raise UndeclaredEventEmissionError unless self.class.emits_event?(event.class)
+        event.body.merge!(DRIVEN_BY_EVENT_PAYLOAD_KEY => _event.id)
+        invoke_action_and_emit_event(event, block)
+        EventSourcery.logger.debug { "[#{self.processor_name}] Emitted event: #{event.inspect}" }
+      end
+
+      def invoke_action_and_emit_event(event, action)
+        action.call(event.body) if action
+        event_sink.sink(event)
+      end
+    end
+  end
+end

data/lib/event_sourcery/postgres/schema.rb

@@ -0,0 +1,150 @@
+module EventSourcery
+  module Postgres
+    module Schema
+      extend self
+
+      def create_event_store(db: EventSourcery::Postgres.config.event_store_database,
+                             events_table_name: EventSourcery::Postgres.config.events_table_name,
+                             aggregates_table_name: EventSourcery::Postgres.config.aggregates_table_name,
+                             write_events_function_name: EventSourcery::Postgres.config.write_events_function_name)
+        create_events(db: db, table_name: events_table_name)
+        create_aggregates(db: db, table_name: aggregates_table_name)
+        create_or_update_functions(db: db, events_table_name: events_table_name, function_name: write_events_function_name, aggregates_table_name: aggregates_table_name)
+      end
+
+      def create_events(db: EventSourcery::Postgres.config.event_store_database,
+                        table_name: EventSourcery::Postgres.config.events_table_name)
+        db.run 'CREATE EXTENSION IF NOT EXISTS "uuid-ossp"'
+        db.create_table(table_name) do
+          primary_key :id, type: :Bignum
+          column :uuid,           :uuid,    null: false, default: Sequel.lit('uuid_generate_v4()')
+          column :aggregate_id,   :uuid,    null: false
+          column :type,           :varchar, null: false, size: 255
+          column :body,           :json,    null: false
+          column :version,        :bigint,  null: false
+          column :correlation_id, :uuid
+          column :causation_id,   :uuid
+          column :created_at,     :'timestamp without time zone', null: false, default: Sequel.lit("(now() at time zone 'utc')")
+          index [:aggregate_id, :version], unique: true
+          index :uuid, unique: true
+          index :type
+          index :created_at
+        end
+      end
+
+      def create_aggregates(db: EventSourcery::Postgres.config.event_store_database,
+                            table_name: EventSourcery::Postgres.config.aggregates_table_name)
+        db.create_table(table_name) do
+          primary_key :aggregate_id, :uuid
+          column :version, :bigint, default: 1
+        end
+      end
+
+      def create_or_update_functions(db: EventSourcery::Postgres.config.event_store_database,
+                                     function_name: EventSourcery::Postgres.config.write_events_function_name,
+                                     events_table_name: EventSourcery::Postgres.config.events_table_name,
+                                     aggregates_table_name: EventSourcery::Postgres.config.aggregates_table_name)
+        db.run <<-SQL
+create or replace function #{function_name}(_aggregateId uuid,
+                                            _eventTypes varchar[],
+                                            _expectedVersion int,
+                                            _bodies json[],
+                                            _createdAtTimes timestamp without time zone[],
+                                            _eventUUIDs uuid[],
+                                            _correlationIds uuid[],
+                                            _causationIds uuid[],
+                                            _lockTable boolean) returns void as $$
+declare
+currentVersion int;
+body json;
+eventVersion int;
+eventId text;
+index int;
+newVersion int;
+numEvents int;
+createdAt timestamp without time zone;
+begin
+numEvents := array_length(_bodies, 1);
+select version into currentVersion from #{aggregates_table_name} where aggregate_id = _aggregateId;
+if not found then
+  -- when we have no existing version for this aggregate
+  if _expectedVersion = 0 or _expectedVersion is null then
+    -- set the version to 1 if expected version is null or 0
+    insert into #{aggregates_table_name}(aggregate_id, version) values(_aggregateId, numEvents);
+    currentVersion := 0;
+  else
+    raise 'Concurrency conflict. Current version: 0, expected version: %', _expectedVersion;
+  end if;
+else
+  if _expectedVersion is null then
+    -- automatically increment the version
+    update #{aggregates_table_name} set version = version + numEvents where aggregate_id = _aggregateId returning version into newVersion;
+    currentVersion := newVersion - numEvents;
+  else
+    -- increment the version if it's at our expected version
+    update #{aggregates_table_name} set version = version + numEvents where aggregate_id = _aggregateId and version = _expectedVersion;
+    if not found then
+      -- version was not at expected_version, raise an error.
+      -- currentVersion may not equal what it did in the database when the
+      -- above update statement is executed (it may have been incremented by another
+      -- process)
+      raise 'Concurrency conflict. Last known current version: %, expected version: %', currentVersion, _expectedVersion;
+    end if;
+  end if;
+end if;
+index := 1;
+eventVersion := currentVersion + 1;
+if _lockTable then
+    -- Ensure this transaction is the only one writing events to guarantee
+    -- linear growth of sequence IDs.
+    -- Any value that won't conflict with other advisory locks will work.
+    -- The Postgres tracker currently obtains an advisory lock using it's
+    -- integer row ID, so values 1 to the number of ESP's in the system would
+    -- be taken if the tracker is running in the same database as your
+    -- projections.
+    perform pg_advisory_xact_lock(-1);
+end if;
+foreach body IN ARRAY(_bodies)
+loop
+  if _createdAtTimes[index] is not null then
+    createdAt := _createdAtTimes[index];
+  else
+    createdAt := now() at time zone 'utc';
+  end if;
+
+  insert into #{events_table_name}
+    (uuid, aggregate_id, type, body, version, correlation_id, causation_id, created_at)
+  values
+    (
+      _eventUUIDs[index],
+      _aggregateId,
+      _eventTypes[index],
+      body,
+      eventVersion,
+      _correlationIds[index],
+      _causationIds[index],
+      createdAt
+    )
+  returning id into eventId;
+
+  eventVersion := eventVersion + 1;
+  index := index + 1;
+end loop;
+perform pg_notify('new_event', eventId);
+end;
+$$ language plpgsql;
+SQL
+      end
+
+      def create_projector_tracker(db: EventSourcery::Postgres.config.projections_database,
+                                   table_name: EventSourcery::Postgres.config.tracker_table_name)
+        db.create_table(table_name) do
+          primary_key :id, type: :Bignum
+          column :name, 'varchar(255) not null'
+          column :last_processed_event_id, 'bigint not null default 0'
+          index :name, unique: true
+        end
+      end
+    end
+  end
+end

data/lib/event_sourcery/postgres/table_owner.rb

@@ -0,0 +1,74 @@
+module EventSourcery
+  module Postgres
+    module TableOwner
+      DefaultTableError = Class.new(StandardError)
+      NoSuchTableError = Class.new(StandardError)
+
+      def self.prepended(base)
+        base.extend(ClassMethods)
+      end
+
+      module ClassMethods
+        def tables
+          @tables ||= {}
+        end
+
+        def table(name, &block)
+          tables[name] = block
+        end
+      end
+
+      def setup
+        self.class.tables.each do |table_name, schema_block|
+          prefixed_name = table_name_prefixed(table_name)
+          @db_connection.create_table?(prefixed_name, &schema_block)
+        end
+        super if defined?(super)
+      end
+
+      def reset
+        self.class.tables.keys.each do |table_name|
+          prefixed_name = table_name_prefixed(table_name)
+          if @db_connection.table_exists?(prefixed_name)
+            @db_connection.drop_table(prefixed_name, cascade: true)
+          end
+        end
+        super if defined?(super)
+        setup
+      end
+
+      def truncate
+        self.class.tables.each do |table_name, _|
+          @db_connection.transaction do
+            prefixed_name = table_name_prefixed(table_name)
+            @db_connection[prefixed_name].truncate
+            tracker.reset_last_processed_event_id(self.class.processor_name)
+          end
+        end
+      end
+
+      private
+
+      attr_reader :db_connection
+      attr_accessor :table_prefix
+
+      def table(name = nil)
+        if name.nil? && self.class.tables.length != 1
+          raise DefaultTableError, 'You must specify table name when when 0 or multiple tables are defined'
+        end
+
+        name ||= self.class.tables.keys.first
+
+        unless self.class.tables[name.to_sym]
+          raise NoSuchTableError, "There is no table with the name '#{name}' defined"
+        end
+
+        db_connection[table_name_prefixed(name)]
+      end
+
+      def table_name_prefixed(name)
+        [table_prefix, name].compact.join("_").to_sym
+      end
+    end
+  end
+end