event_sourcery-postgres 0.3.0

data/lib/event_sourcery/postgres/tracker.rb

@@ -0,0 +1,90 @@
+module EventSourcery
+  module Postgres
+    class Tracker
+      def initialize(connection = EventSourcery::Postgres.config.projections_database,
+                     table_name: EventSourcery::Postgres.config.tracker_table_name,
+                     obtain_processor_lock: true)
+        @connection = connection
+        @table_name = table_name
+        @obtain_processor_lock = obtain_processor_lock
+      end
+
+      def setup(processor_name = nil)
+        create_table_if_not_exists if EventSourcery::Postgres.config.auto_create_projector_tracker
+
+        unless tracker_table_exists?
+          raise UnableToLockProcessorError, "Projector tracker table does not exist"
+        end
+
+        if processor_name
+          create_track_entry_if_not_exists(processor_name)
+          if @obtain_processor_lock
+            obtain_global_lock_on_processor(processor_name)
+          end
+        end
+      end
+
+      def processed_event(processor_name, event_id)
+        table.
+          where(name: processor_name.to_s).
+                update(last_processed_event_id: event_id)
+        true
+      end
+
+      def processing_event(processor_name, event_id)
+        @connection.transaction do
+          yield
+          processed_event(processor_name, event_id)
+        end
+      end
+
+      def reset_last_processed_event_id(processor_name)
+        table.where(name: processor_name.to_s).update(last_processed_event_id: 0)
+      end
+
+      def last_processed_event_id(processor_name)
+        track_entry = table.where(name: processor_name.to_s).first
+        if track_entry
+          track_entry[:last_processed_event_id]
+        end
+      end
+
+      def tracked_processors
+        table.select_map(:name)
+      end
+
+      private
+
+      def obtain_global_lock_on_processor(processor_name)
+        lock_obtained = @connection.fetch("select pg_try_advisory_lock(#{@track_entry_id})").to_a.first[:pg_try_advisory_lock]
+        if lock_obtained == false
+          raise UnableToLockProcessorError, "Unable to get a lock on #{processor_name} #{@track_entry_id}"
+        end
+      end
+
+      def create_table_if_not_exists
+        unless tracker_table_exists?
+          EventSourcery.logger.info { "Projector tracker missing - attempting to create 'projector_tracker' table" }
+          EventSourcery::Postgres::Schema.create_projector_tracker(db: @connection, table_name: @table_name)
+        end
+      end
+
+      def create_track_entry_if_not_exists(processor_name)
+        track_entry = table.where(name: processor_name.to_s).first
+        @track_entry_id = if track_entry
+                            track_entry[:id]
+                          else
+                            table.insert(name: processor_name.to_s, last_processed_event_id: 0)
+                          end
+      end
+
+      def table
+        @connection[@table_name]
+      end
+
+      def tracker_table_exists?
+        @connection.table_exists?(@table_name)
+      end
+    end
+  end
+end

data/lib/event_sourcery/postgres/version.rb

@@ -0,0 +1,5 @@
+module EventSourcery
+  module Postgres
+    VERSION = "0.3.0"
+  end
+end

data/script/bench_reading_events.rb

@@ -0,0 +1,63 @@
+# Usage:
+#
+# ❯ bundle exec ruby script/bench_reading_events.rb
+# Creating 10000 events
+# Took 42.35533199999918 to create events
+# Took 4.9821800000027 to read all events
+# ^ results from running on a 2016 MacBook
+
+require 'benchmark'
+require 'securerandom'
+require 'sequel'
+require 'event_sourcery/postgres'
+
+pg_uri = ENV.fetch('BOXEN_POSTGRESQL_URL') { 'postgres://127.0.0.1:5432/' }.dup
+pg_uri << 'event_sourcery_test'
+pg_connection = Sequel.connect(pg_uri)
+
+EventSourcery.configure do |config|
+  config.postgres.event_store_database = pg_connection
+  config.postgres.projections_database = pg_connection
+  config.logger.level = :fatal
+end
+
+def create_events_schema(pg_connection)
+  pg_connection.execute 'drop table if exists events'
+  pg_connection.execute 'drop table if exists aggregates'
+  EventSourcery::Postgres::Schema.create_event_store(db: pg_connection)
+end
+
+event_store = EventSourcery::Postgres.config.event_store
+
+EVENT_TYPES = %i[
+  item_added
+  item_removed
+  item_starred
+]
+
+def new_event(uuid)
+  EventSourcery::Event.new(type: EVENT_TYPES.sample,
+                           aggregate_id: uuid,
+                           body: { 'something' => 'simple' })
+end
+
+create_events_schema(pg_connection)
+
+NUM_EVENTS = 10_000
+puts "Creating #{NUM_EVENTS} events"
+time = Benchmark.realtime do
+  uuid = SecureRandom.uuid
+  NUM_EVENTS.times do
+    event_store.sink(new_event(uuid))
+  end
+end
+puts "Took #{time} to create events"
+
+seen_events_count = 0
+time = Benchmark.realtime do
+  event_store.subscribe(from_id: 0, subscription_master: EventSourcery::EventStore::SignalHandlingSubscriptionMaster.new) do |events|
+    seen_events_count += events.count
+    throw :stop if seen_events_count >= NUM_EVENTS
+  end
+end
+puts "Took #{time} to read all events"

data/script/bench_writing_events.rb

@@ -0,0 +1,47 @@
+# Usage:
+#
+# ❯ bundle exec ruby script/bench_writing_events.rb
+# Warming up --------------------------------------
+# event_store.sink
+#                         70.000  i/100ms
+# Calculating -------------------------------------
+# event_store.sink
+#                         522.007  (±10.9%) i/s -      2.590k in   5.021909s
+#
+# ^ results from running on a 2016 MacBook
+
+require 'benchmark/ips'
+require 'securerandom'
+require 'sequel'
+require 'event_sourcery/postgres'
+
+pg_uri = ENV.fetch('BOXEN_POSTGRESQL_URL') { 'postgres://127.0.0.1:5432/' }.dup
+pg_uri << 'event_sourcery_test'
+pg_connection = Sequel.connect(pg_uri)
+
+EventSourcery.configure do |config|
+  config.postgres.event_store_database = pg_connection
+  config.postgres.projections_database = pg_connection
+  config.logger.level = :fatal
+end
+
+def create_schema(pg_connection)
+  pg_connection.execute 'drop table if exists events'
+  pg_connection.execute 'drop table if exists aggregates'
+  EventSourcery::Postgres::Schema.create_event_store(db: pg_connection)
+end
+
+create_schema(pg_connection)
+event_store = EventSourcery::Postgres::EventStore.new(pg_connection)
+
+def new_event
+  EventSourcery::Event.new(type: :item_added,
+                           aggregate_id: SecureRandom.uuid,
+                           body: { 'something' => 'simple' })
+end
+
+Benchmark.ips do |b|
+  b.report("event_store.sink") do
+    event_store.sink(new_event)
+  end
+end

data/script/demonstrate_event_sequence_id_gaps.rb

@@ -0,0 +1,181 @@
+# Demonstrates that sequence IDs may not be inserted linearly with concurrent
+# writers.
+#
+# This script writes events in parallel from a number of forked processes,
+# writing events in a continious loop until the program is interrupted.
+# The parent process detects gaps in sequence IDs by selecting the last 2
+# events based on sequence ID. A gap is detected when the 2 IDs returned from
+# that query aren't sequential. The script will proceed to execute 2 subsequent
+# queries to see if they show up in the time it takes to complete those before
+# moving on.
+#
+# An easier way to demonstrate this is by using 2 psql consoles:
+#
+# - Simulate a transaction taking a long time to commit:
+#   ```
+#   begin;
+#   insert into events (..) values (..);
+#   ```
+# - Then, in another console:
+#   ```
+#   insert into events (..) values (..);
+#   select * from events;
+#   ```
+#
+# The result is that event sequence ID 2 is visible, but only when the first
+# transaction commits is event sequence ID 1 visible.
+#
+# Why does this happen?
+#
+# Sequences in Postgres (and most other DBs) are not transactional, changes
+# to the sequence are visible to other transactions immediately. Also, inserts
+# from the forked writers may be executed in parallel by postgres.
+#
+# The process of inserting into a table that has a sequence or serial column is
+# to first get the next sequence ID (changing global state), then perform the
+# insert statement and later commit. In between these 2 steps the sequence ID
+# is taken but not visible in the table until the insert statement is
+# committed. Gaps in sequence IDs occur when a process takes a sequence ID and
+# commits it while another process is in between those 2 steps.
+#
+# This means another transaction could have taken the next sequence
+# ID and committed before that one commits, resulting in a gap in sequence ID's
+# when reading.
+#
+# Why is this a problem?
+#
+# Event stream processors use the sequence ID to keep track of where they're up to
+# in the events table. If a projector processes an event with sequence ID n, it
+# assumes that the next event it needs to process will have a sequence ID > n.
+# This approach isn't reliable when sequence IDs appear non-linearly, making it
+# possible for event stream processors to skip over events.
+#
+# How does EventSourcery deal with this?
+#
+# EventSourcery uses n transaction level advisory lock to synchronise inserts
+# to the events table within the writeEvents function. Alternatives:
+#
+# - Write events from 1 process only (serialize at the application level)
+# - Detect gaps when reading events and allow time for in-flight transactions
+# (the gaps) to commit.
+# - Built in eventual consistency. Selects would be restricted to events older
+# than 500ms-1s or the transaction timeout to give enough time for in-flight
+# transactions to commit.
+# - Only query events when catching up, after that rely on events to be
+# delivered through the pub/sub mechanism. Given events would be received out
+# of order under concurrent writes there's potential for processors to process
+# a given event twice if they shutdown after processing a sequence that was
+# part of a gap.
+#
+# Usage:
+#
+# ❯ bundle exec ruby script/demonstrate_event_sequence_id_gaps.rb
+# 89847: starting to write events89846: starting to write events
+
+# 89848: starting to write events
+# 89849: starting to write events
+# 89850: starting to write events
+# GAP: 1 missing sequence IDs. 78 != 76 + 1. Missing events showed up after 1 subsequent query. IDs: [77]
+# GAP: 1 missing sequence IDs. 168 != 166 + 1. Missing events showed up after 1 subsequent query. IDs: [167]
+# GAP: 1 missing sequence IDs. 274 != 272 + 1. Missing events showed up after 1 subsequent query. IDs: [273]
+# GAP: 1 missing sequence IDs. 341 != 339 + 1. Missing events showed up after 1 subsequent query. IDs: [340]
+# GAP: 1 missing sequence IDs. 461 != 459 + 1. Missing events showed up after 1 subsequent query. IDs: [460]
+# GAP: 1 missing sequence IDs. 493 != 491 + 1. Missing events showed up after 1 subsequent query. IDs: [492]
+# GAP: 2 missing sequence IDs. 621 != 618 + 1. Missing events showed up after 1 subsequent query. IDs: [619, 620]
+
+require 'sequel'
+require 'securerandom'
+require 'event_sourcery/postgres'
+
+def connect
+  pg_uri = ENV.fetch('BOXEN_POSTGRESQL_URL') { 'postgres://127.0.0.1:5432/' }.dup
+  pg_uri << 'event_sourcery_test'
+  Sequel.connect(pg_uri)
+end
+
+EventSourcery.logger.level = :info
+
+def new_event
+  EventSourcery::Event.new(type: :item_added,
+                           aggregate_id: SecureRandom.uuid,
+                           body: { 'something' => 'simple' })
+end
+
+def create_events_schema(db)
+  db.execute 'drop table if exists events'
+  db.execute 'drop table if exists aggregates'
+  EventSourcery::Postgres::Schema.create_event_store(db: db)
+end
+
+db = connect
+create_events_schema(db)
+db.disconnect
+sleep 0.3
+
+NUM_WRITER_PROCESSES = 5
+NUM_WRITER_PROCESSES.times do
+  fork do |pid|
+    stop = false
+    Signal.trap(:INT) { stop = true }
+    db = connect
+    # when lock_table is set to true an advisory lock is used to synchronise
+    # inserts and no gaps are detected
+    event_store = EventSourcery::Postgres::EventStore.new(db, lock_table: false)
+    puts "#{Process.pid}: starting to write events"
+    until stop
+      event_store.sink(new_event)
+    end
+  end
+end
+
+stop = false
+Signal.trap(:INT) { stop = true }
+
+def wait_for_missing_ids(db, first_sequence, last_sequence, attempt: 1)
+  missing_ids = db[:events].where(Sequel.lit("id > ? AND id < ?", first_sequence, last_sequence)).order(:id).map {|e| e[:id] }
+  expected_missing_ids = (first_sequence+1)..(last_sequence-1)
+  if missing_ids == expected_missing_ids.to_a
+    print "Missing events showed up after #{attempt} subsequent query. IDs: #{missing_ids}"
+  else
+    if attempt < 2
+      wait_for_missing_ids(db, first_sequence, last_sequence, attempt: attempt + 1)
+    else
+      print "Missing events didn't show up after #{attempt} subsequent queries"
+    end
+  end
+end
+
+until stop
+
+  # query for the last 2 sequences in the events table
+  first_sequence, last_sequence = *db[:events].
+    order(Sequel.desc(:id)).
+    select(:id).
+    limit(2).
+    map { |e| e[:id] }.
+    reverse
+
+  next if first_sequence.nil? || last_sequence.nil?
+
+  if last_sequence != first_sequence + 1
+    num_missing = last_sequence - first_sequence - 1
+    print "GAP: #{num_missing} missing sequence IDs. #{last_sequence} != #{first_sequence} + 1. "
+    wait_for_missing_ids(db, first_sequence, last_sequence)
+    puts
+  end
+end
+
+Process.waitall
+
+puts
+puts "Looking for gaps in sequence IDs in events table:"
+ids = db[:events].select(:id).order(:id).all.map { |e| e[:id] }
+expected_ids = (ids.min..ids.max).to_a
+missing_ids = (expected_ids - ids)
+if missing_ids.empty?
+  puts "No remaining gaps"
+else
+  missing_ids.each do |id|
+    puts "Unable to find row with sequence ID #{id}"
+  end
+end

metadata

@@ -0,0 +1,181 @@
+--- !ruby/object:Gem::Specification
+name: event_sourcery-postgres
+version: !ruby/object:Gem::Version
+  version: 0.3.0
+platform: ruby
+authors:
+- Steve Hodgkiss
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-06-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.38'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.38'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: event_sourcery
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: benchmark-ips
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- steve@hodgkiss.me
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- CHANGELOG.md
+- CODE_OF_CONDUCT.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- event_sourcery-postgres.gemspec
+- lib/event_sourcery/postgres.rb
+- lib/event_sourcery/postgres/config.rb
+- lib/event_sourcery/postgres/event_store.rb
+- lib/event_sourcery/postgres/optimised_event_poll_waiter.rb
+- lib/event_sourcery/postgres/projector.rb
+- lib/event_sourcery/postgres/queue_with_interval_callback.rb
+- lib/event_sourcery/postgres/reactor.rb
+- lib/event_sourcery/postgres/schema.rb
+- lib/event_sourcery/postgres/table_owner.rb
+- lib/event_sourcery/postgres/tracker.rb
+- lib/event_sourcery/postgres/version.rb
+- script/bench_reading_events.rb
+- script/bench_writing_events.rb
+- script/demonstrate_event_sequence_id_gaps.rb
+homepage: https://github.com/envato/event_sourcery-postgres
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.2.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.11
+signing_key: 
+specification_version: 4
+summary: Postgres event store for use with EventSourcery
+test_files: []