event_sourcery-postgres 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 75bff88c421cb623d91957ca0afec6054a835f2a
4
- data.tar.gz: dd7c0cd3eaf82c26bb328fbbcdd80e810a38b45b
3
+ metadata.gz: e45ae5ad105bd165a2fb4b59be6032488ec925df
4
+ data.tar.gz: c36491b6e980803520d2b3a900207500f694b500
5
5
  SHA512:
6
- metadata.gz: 016f2f36b96fef639e26584458ec2798649d3d81f54ce39a398fb4eec5645a31c278d680134630239cf3efba9ee76cccca5117dd98d56b59d8a8a6ce5b6c2efc
7
- data.tar.gz: c033c7b72cb9ea7c31de75aca7c51c168397ea1eb18703eae709a2262a4f9a84893b740195acb5023fab05433fdaa9db26780232ef4ec416d579d3795cc00884
6
+ metadata.gz: a86c88d7db9e0368d3ddbea1b6b7ce869d8f3465dd1bce3990f0038edffcb5712da10d4182fd0ce46427b2e3aa08b67b7da6712d7aec0abd16f766741a492556
7
+ data.tar.gz: 3f286cd47b3eb36e3b3e65929d6df0d51a1bc8762f1256d1b21c6f8041861fc8087bde5978989c7abb58d6f274add9dca368ca992105a12e5e7af6069e390dc7
data/CHANGELOG.md CHANGED
@@ -6,6 +6,19 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
6
6
 
7
7
  ## [Unreleased]
8
8
 
9
+ ## [0.4.0] - 2017-6-21
10
+ ### Changed
11
+ - Reactors store the UUID of the event being processed in the `causation_id`
12
+ of any emitted events. This replaces the old behaviour of storing id of the
13
+ event being processed in a `_driven_by_event_id` attribute in the emitted
14
+ event's body.
15
+
16
+ ### Added
17
+ - Reactors store the correlation id of the event being processed in the
18
+ `correlation_id` of any emitted events.
19
+ - Added index on the `events` table for `correlation_id` and `causation_id`
20
+ columns.
21
+
9
22
  ## [0.3.0] - 2017-6-16
10
23
  ### Changed
11
24
  - The event store persists the event `correlation_id` and `causation_id`.
@@ -4,30 +4,30 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
  require 'event_sourcery/postgres/version'
5
5
 
6
6
  Gem::Specification.new do |spec|
7
- spec.name = "event_sourcery-postgres"
7
+ spec.name = 'event_sourcery-postgres'
8
8
  spec.version = EventSourcery::Postgres::VERSION
9
- # TODO: update authors
10
- spec.authors = ["Steve Hodgkiss"]
11
- spec.email = ["steve@hodgkiss.me"]
12
9
 
13
- spec.summary = %q{Postgres event store for use with EventSourcery}
14
- spec.homepage = "https://github.com/envato/event_sourcery-postgres"
10
+ spec.authors = ['Envato']
11
+ spec.email = ['rubygems@envato.com']
12
+
13
+ spec.summary = 'Postgres event store for use with EventSourcery'
14
+ spec.homepage = 'https://github.com/envato/event_sourcery-postgres'
15
15
 
16
16
  spec.files = `git ls-files -z`.split("\x0").reject do |f|
17
- f.match(%r{^(test|spec|features)/})
17
+ f.match(%r{^(\.|bin/|Gemfile|Rakefile|script/|spec/)})
18
18
  end
19
- spec.bindir = "exe"
19
+ spec.bindir = 'exe'
20
20
  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
21
- spec.require_paths = ["lib"]
21
+ spec.require_paths = ['lib']
22
22
 
23
23
  spec.required_ruby_version = '>= 2.2.0'
24
24
 
25
25
  spec.add_dependency 'sequel', '~> 4.38'
26
26
  spec.add_dependency 'pg'
27
- spec.add_dependency 'event_sourcery', '>= 0.10.0'
28
- spec.add_development_dependency "bundler", "~> 1.10"
29
- spec.add_development_dependency "rake", "~> 10.0"
30
- spec.add_development_dependency "rspec", "~> 3.0"
31
- spec.add_development_dependency "pry"
32
- spec.add_development_dependency "benchmark-ips"
27
+ spec.add_dependency 'event_sourcery', '>= 0.14.0'
28
+ spec.add_development_dependency 'bundler', '~> 1.10'
29
+ spec.add_development_dependency 'rake', '~> 10.0'
30
+ spec.add_development_dependency 'rspec', '~> 3.0'
31
+ spec.add_development_dependency 'pry'
32
+ spec.add_development_dependency 'benchmark-ips'
33
33
  end
@@ -36,19 +36,13 @@ module EventSourcery
36
36
  order(:id).
37
37
  where(Sequel.lit('id >= ?', id)).
38
38
  limit(limit)
39
- if event_types
40
- query = query.where(type: event_types)
41
- end
42
- query.map do |event_row|
43
- build_event(event_row)
44
- end
39
+ query = query.where(type: event_types) if event_types
40
+ query.map { |event_row| build_event(event_row) }
45
41
  end
46
42
 
47
43
  def latest_event_id(event_types: nil)
48
44
  latest_event = events_table
49
- if event_types
50
- latest_event = latest_event.where(type: event_types)
51
- end
45
+ latest_event = latest_event.where(type: event_types) if event_types
52
46
  latest_event = latest_event.order(:id).last
53
47
  if latest_event
54
48
  latest_event[:id]
@@ -74,9 +68,7 @@ module EventSourcery
74
68
  subscription_master: subscription_master,
75
69
  on_new_events: block
76
70
  }
77
- EventSourcery::EventStore::Subscription.new(args).tap do |s|
78
- s.start
79
- end
71
+ EventSourcery::EventStore::Subscription.new(args).tap(&:start)
80
72
  end
81
73
 
82
74
  private
@@ -113,7 +105,7 @@ module EventSourcery
113
105
 
114
106
  def sql_literal_array(events, type, &block)
115
107
  sql_array = events.map do |event|
116
- to_sql_literal(block.call(event))
108
+ to_sql_literal(block.call(event))
117
109
  end.join(', ')
118
110
  "array[#{sql_array}]::#{type}[]"
119
111
  end
@@ -125,12 +117,12 @@ module EventSourcery
125
117
  def to_sql_literal(value)
126
118
  return 'null' unless value
127
119
  wrapped_value = if Time === value
128
- value.iso8601(6)
129
- elsif Hash === value
130
- Sequel.pg_json(value)
131
- else
132
- value
133
- end
120
+ value.iso8601(6)
121
+ elsif Hash === value
122
+ Sequel.pg_json(value)
123
+ else
124
+ value
125
+ end
134
126
  @pg_connection.literal(wrapped_value)
135
127
  end
136
128
 
@@ -4,7 +4,7 @@ module EventSourcery
4
4
  class OptimisedEventPollWaiter
5
5
  ListenThreadDied = Class.new(StandardError)
6
6
 
7
- def initialize(pg_connection:, timeout: 30, after_listen: proc { })
7
+ def initialize(pg_connection:, timeout: 30, after_listen: proc {})
8
8
  @pg_connection = pg_connection
9
9
  @timeout = timeout
10
10
  @events_queue = QueueWithIntervalCallback.new
@@ -17,7 +17,7 @@ module EventSourcery
17
17
  block.call
18
18
  end
19
19
  start_async(after_listen: after_listen)
20
- catch(:stop) {
20
+ catch(:stop) do
21
21
  block.call
22
22
  loop do
23
23
  ensure_listen_thread_alive!
@@ -25,7 +25,7 @@ module EventSourcery
25
25
  clear_new_event_queue
26
26
  block.call
27
27
  end
28
- }
28
+ end
29
29
  ensure
30
30
  shutdown!
31
31
  end
@@ -33,15 +33,11 @@ module EventSourcery
33
33
  private
34
34
 
35
35
  def shutdown!
36
- if @listen_thread.alive?
37
- @listen_thread.kill
38
- end
36
+ @listen_thread.kill if @listen_thread.alive?
39
37
  end
40
38
 
41
39
  def ensure_listen_thread_alive!
42
- if !@listen_thread.alive?
43
- raise ListenThreadDied
44
- end
40
+ raise ListenThreadDied unless @listen_thread.alive?
45
41
  end
46
42
 
47
43
  def wait_for_new_event_to_appear
@@ -54,16 +50,18 @@ module EventSourcery
54
50
 
55
51
  def start_async(after_listen: nil)
56
52
  after_listen_callback = if after_listen
57
- proc {
53
+ proc do
58
54
  after_listen.call
59
55
  @after_listen.call if @after_listen
60
- }
56
+ end
61
57
  else
62
58
  @after_listen
63
59
  end
64
- @listen_thread = Thread.new { listen_for_new_events(loop: true,
65
- after_listen: after_listen_callback,
66
- timeout: @timeout) }
60
+ @listen_thread = Thread.new do
61
+ listen_for_new_events(loop: true,
62
+ after_listen: after_listen_callback,
63
+ timeout: @timeout)
64
+ end
67
65
  end
68
66
 
69
67
  def listen_for_new_events(loop: true, after_listen: nil, timeout: 30)
@@ -71,9 +69,7 @@ module EventSourcery
71
69
  loop: loop,
72
70
  after_listen: after_listen,
73
71
  timeout: timeout) do |_channel, _pid, _payload|
74
- if @events_queue.empty?
75
- @events_queue.push(:new_event_arrived)
76
- end
72
+ @events_queue.push(:new_event_arrived) if @events_queue.empty?
77
73
  end
78
74
  end
79
75
  end
@@ -6,12 +6,12 @@ module EventSourcery
6
6
  base.prepend(TableOwner)
7
7
  base.include(InstanceMethods)
8
8
  base.class_eval do
9
- alias project process
9
+ alias_method :project, :process
10
10
 
11
11
  class << self
12
- alias project process
13
- alias projects_events processes_events
14
- alias projector_name processor_name
12
+ alias_method :project, :process
13
+ alias_method :projects_events, :processes_events
14
+ alias_method :projector_name, :processor_name
15
15
  end
16
16
  end
17
17
  end
@@ -3,7 +3,7 @@ module EventSourcery
3
3
  class QueueWithIntervalCallback < ::Queue
4
4
  attr_accessor :callback
5
5
 
6
- def initialize(callback: proc { }, callback_interval: EventSourcery::Postgres.config.callback_interval_if_no_new_events, poll_interval: 0.1)
6
+ def initialize(callback: proc {}, callback_interval: EventSourcery::Postgres.config.callback_interval_if_no_new_events, poll_interval: 0.1)
7
7
  @callback = callback
8
8
  @callback_interval = callback_interval
9
9
  @poll_interval = poll_interval
@@ -20,7 +20,7 @@ module EventSourcery
20
20
  def pop_with_interval_callback
21
21
  time = Time.now
22
22
  loop do
23
- return pop(true) if !empty?
23
+ return pop(true) unless empty?
24
24
  if @callback_interval && Time.now > time + @callback_interval
25
25
  @callback.call
26
26
  time = Time.now
@@ -45,20 +45,18 @@ module EventSourcery
45
45
  end
46
46
  end
47
47
 
48
- DRIVEN_BY_EVENT_PAYLOAD_KEY = :_driven_by_event_id
49
-
50
48
  private
51
49
 
52
50
  attr_reader :event_sink, :event_source
53
51
 
54
52
  def emit_event(event_or_hash, &block)
55
53
  event = if Event === event_or_hash
56
- event_or_hash
57
- else
58
- Event.new(event_or_hash)
59
- end
54
+ event_or_hash
55
+ else
56
+ Event.new(event_or_hash)
57
+ end
60
58
  raise UndeclaredEventEmissionError unless self.class.emits_event?(event.class)
61
- event.body.merge!(DRIVEN_BY_EVENT_PAYLOAD_KEY => _event.id)
59
+ event = event.with(causation_id: _event.uuid, correlation_id: _event.correlation_id)
62
60
  invoke_action_and_emit_event(event, block)
63
61
  EventSourcery.logger.debug { "[#{self.processor_name}] Emitted event: #{event.inspect}" }
64
62
  end
@@ -1,7 +1,7 @@
1
1
  module EventSourcery
2
2
  module Postgres
3
3
  module Schema
4
- extend self
4
+ module_function
5
5
 
6
6
  def create_event_store(db: EventSourcery::Postgres.config.event_store_database,
7
7
  events_table_name: EventSourcery::Postgres.config.events_table_name,
@@ -28,6 +28,8 @@ module EventSourcery
28
28
  index [:aggregate_id, :version], unique: true
29
29
  index :uuid, unique: true
30
30
  index :type
31
+ index :correlation_id
32
+ index :causation_id
31
33
  index :created_at
32
34
  end
33
35
  end
@@ -67,7 +67,7 @@ module EventSourcery
67
67
  end
68
68
 
69
69
  def table_name_prefixed(name)
70
- [table_prefix, name].compact.join("_").to_sym
70
+ [table_prefix, name].compact.join('_').to_sym
71
71
  end
72
72
  end
73
73
  end
@@ -5,7 +5,7 @@ module EventSourcery
5
5
  table_name: EventSourcery::Postgres.config.tracker_table_name,
6
6
  obtain_processor_lock: true)
7
7
  @connection = connection
8
- @table_name = table_name
8
+ @table_name = table_name.to_sym
9
9
  @obtain_processor_lock = obtain_processor_lock
10
10
  end
11
11
 
@@ -13,7 +13,7 @@ module EventSourcery
13
13
  create_table_if_not_exists if EventSourcery::Postgres.config.auto_create_projector_tracker
14
14
 
15
15
  unless tracker_table_exists?
16
- raise UnableToLockProcessorError, "Projector tracker table does not exist"
16
+ raise UnableToLockProcessorError, 'Projector tracker table does not exist'
17
17
  end
18
18
 
19
19
  if processor_name
@@ -27,7 +27,7 @@ module EventSourcery
27
27
  def processed_event(processor_name, event_id)
28
28
  table.
29
29
  where(name: processor_name.to_s).
30
- update(last_processed_event_id: event_id)
30
+ update(last_processed_event_id: event_id)
31
31
  true
32
32
  end
33
33
 
@@ -44,9 +44,7 @@ module EventSourcery
44
44
 
45
45
  def last_processed_event_id(processor_name)
46
46
  track_entry = table.where(name: processor_name.to_s).first
47
- if track_entry
48
- track_entry[:last_processed_event_id]
49
- end
47
+ track_entry[:last_processed_event_id] if track_entry
50
48
  end
51
49
 
52
50
  def tracked_processors
@@ -1,5 +1,5 @@
1
1
  module EventSourcery
2
2
  module Postgres
3
- VERSION = "0.3.0"
3
+ VERSION = '0.4.0'.freeze
4
4
  end
5
5
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: event_sourcery-postgres
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.0
4
+ version: 0.4.0
5
5
  platform: ruby
6
6
  authors:
7
- - Steve Hodgkiss
7
+ - Envato
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2017-06-16 00:00:00.000000000 Z
11
+ date: 2017-06-21 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: sequel
@@ -44,14 +44,14 @@ dependencies:
44
44
  requirements:
45
45
  - - ">="
46
46
  - !ruby/object:Gem::Version
47
- version: 0.10.0
47
+ version: 0.14.0
48
48
  type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
52
  - - ">="
53
53
  - !ruby/object:Gem::Version
54
- version: 0.10.0
54
+ version: 0.14.0
55
55
  - !ruby/object:Gem::Dependency
56
56
  name: bundler
57
57
  requirement: !ruby/object:Gem::Requirement
@@ -124,22 +124,15 @@ dependencies:
124
124
  version: '0'
125
125
  description:
126
126
  email:
127
- - steve@hodgkiss.me
127
+ - rubygems@envato.com
128
128
  executables: []
129
129
  extensions: []
130
130
  extra_rdoc_files: []
131
131
  files:
132
- - ".gitignore"
133
- - ".rspec"
134
- - ".travis.yml"
135
132
  - CHANGELOG.md
136
133
  - CODE_OF_CONDUCT.md
137
- - Gemfile
138
134
  - LICENSE.txt
139
135
  - README.md
140
- - Rakefile
141
- - bin/console
142
- - bin/setup
143
136
  - event_sourcery-postgres.gemspec
144
137
  - lib/event_sourcery/postgres.rb
145
138
  - lib/event_sourcery/postgres/config.rb
@@ -152,9 +145,6 @@ files:
152
145
  - lib/event_sourcery/postgres/table_owner.rb
153
146
  - lib/event_sourcery/postgres/tracker.rb
154
147
  - lib/event_sourcery/postgres/version.rb
155
- - script/bench_reading_events.rb
156
- - script/bench_writing_events.rb
157
- - script/demonstrate_event_sequence_id_gaps.rb
158
148
  homepage: https://github.com/envato/event_sourcery-postgres
159
149
  licenses: []
160
150
  metadata: {}
data/.gitignore DELETED
@@ -1,13 +0,0 @@
1
- /.bundle/
2
- /.yardoc
3
- /Gemfile.lock
4
- /_yardoc/
5
- /coverage/
6
- /doc/
7
- /pkg/
8
- /spec/reports/
9
- /tmp/
10
- /vendor/
11
-
12
- # rspec failure tracking
13
- .rspec_status
data/.rspec DELETED
@@ -1,3 +0,0 @@
1
- --require spec_helper
2
- --format documentation
3
- --color
data/.travis.yml DELETED
@@ -1,12 +0,0 @@
1
- sudo: false
2
- language: ruby
3
- rvm:
4
- - 2.2
5
- - 2.3
6
- - 2.4
7
- before_install:
8
- - gem install bundler
9
- before_script:
10
- - psql -c 'create database event_sourcery_test;' -U postgres
11
- addons:
12
- postgresql: 9.4
data/Gemfile DELETED
@@ -1,5 +0,0 @@
1
- source 'https://rubygems.org'
2
-
3
- ruby '>= 2.2.0'
4
-
5
- gemspec
data/Rakefile DELETED
@@ -1,6 +0,0 @@
1
- require "bundler/gem_tasks"
2
- require "rspec/core/rake_task"
3
-
4
- RSpec::Core::RakeTask.new(:spec)
5
-
6
- task :default => :spec
data/bin/console DELETED
@@ -1,14 +0,0 @@
1
- #!/usr/bin/env ruby
2
-
3
- require "bundler/setup"
4
- require "event_sourcery/postgres"
5
-
6
- # You can add fixtures and/or initialization code here to make experimenting
7
- # with your gem easier. You can also use a different console, if you like.
8
-
9
- # (If you use this, don't forget to add pry to your Gemfile!)
10
- # require "pry"
11
- # Pry.start
12
-
13
- require "irb"
14
- IRB.start(__FILE__)
data/bin/setup DELETED
@@ -1,15 +0,0 @@
1
- #!/usr/bin/env bash
2
- set -euo pipefail
3
- IFS=$'\n\t'
4
- set -vx
5
-
6
- bundle install
7
-
8
- echo
9
- echo "--- Preparing test databases"
10
- echo
11
-
12
- dropdb event_sourcery_test || echo 0
13
- createdb event_sourcery_test
14
-
15
- # Do any other automated setup that you need to do here
@@ -1,63 +0,0 @@
1
- # Usage:
2
- #
3
- # ❯ bundle exec ruby script/bench_reading_events.rb
4
- # Creating 10000 events
5
- # Took 42.35533199999918 to create events
6
- # Took 4.9821800000027 to read all events
7
- # ^ results from running on a 2016 MacBook
8
-
9
- require 'benchmark'
10
- require 'securerandom'
11
- require 'sequel'
12
- require 'event_sourcery/postgres'
13
-
14
- pg_uri = ENV.fetch('BOXEN_POSTGRESQL_URL') { 'postgres://127.0.0.1:5432/' }.dup
15
- pg_uri << 'event_sourcery_test'
16
- pg_connection = Sequel.connect(pg_uri)
17
-
18
- EventSourcery.configure do |config|
19
- config.postgres.event_store_database = pg_connection
20
- config.postgres.projections_database = pg_connection
21
- config.logger.level = :fatal
22
- end
23
-
24
- def create_events_schema(pg_connection)
25
- pg_connection.execute 'drop table if exists events'
26
- pg_connection.execute 'drop table if exists aggregates'
27
- EventSourcery::Postgres::Schema.create_event_store(db: pg_connection)
28
- end
29
-
30
- event_store = EventSourcery::Postgres.config.event_store
31
-
32
- EVENT_TYPES = %i[
33
- item_added
34
- item_removed
35
- item_starred
36
- ]
37
-
38
- def new_event(uuid)
39
- EventSourcery::Event.new(type: EVENT_TYPES.sample,
40
- aggregate_id: uuid,
41
- body: { 'something' => 'simple' })
42
- end
43
-
44
- create_events_schema(pg_connection)
45
-
46
- NUM_EVENTS = 10_000
47
- puts "Creating #{NUM_EVENTS} events"
48
- time = Benchmark.realtime do
49
- uuid = SecureRandom.uuid
50
- NUM_EVENTS.times do
51
- event_store.sink(new_event(uuid))
52
- end
53
- end
54
- puts "Took #{time} to create events"
55
-
56
- seen_events_count = 0
57
- time = Benchmark.realtime do
58
- event_store.subscribe(from_id: 0, subscription_master: EventSourcery::EventStore::SignalHandlingSubscriptionMaster.new) do |events|
59
- seen_events_count += events.count
60
- throw :stop if seen_events_count >= NUM_EVENTS
61
- end
62
- end
63
- puts "Took #{time} to read all events"
@@ -1,47 +0,0 @@
1
- # Usage:
2
- #
3
- # ❯ bundle exec ruby script/bench_writing_events.rb
4
- # Warming up --------------------------------------
5
- # event_store.sink
6
- # 70.000 i/100ms
7
- # Calculating -------------------------------------
8
- # event_store.sink
9
- # 522.007 (±10.9%) i/s - 2.590k in 5.021909s
10
- #
11
- # ^ results from running on a 2016 MacBook
12
-
13
- require 'benchmark/ips'
14
- require 'securerandom'
15
- require 'sequel'
16
- require 'event_sourcery/postgres'
17
-
18
- pg_uri = ENV.fetch('BOXEN_POSTGRESQL_URL') { 'postgres://127.0.0.1:5432/' }.dup
19
- pg_uri << 'event_sourcery_test'
20
- pg_connection = Sequel.connect(pg_uri)
21
-
22
- EventSourcery.configure do |config|
23
- config.postgres.event_store_database = pg_connection
24
- config.postgres.projections_database = pg_connection
25
- config.logger.level = :fatal
26
- end
27
-
28
- def create_schema(pg_connection)
29
- pg_connection.execute 'drop table if exists events'
30
- pg_connection.execute 'drop table if exists aggregates'
31
- EventSourcery::Postgres::Schema.create_event_store(db: pg_connection)
32
- end
33
-
34
- create_schema(pg_connection)
35
- event_store = EventSourcery::Postgres::EventStore.new(pg_connection)
36
-
37
- def new_event
38
- EventSourcery::Event.new(type: :item_added,
39
- aggregate_id: SecureRandom.uuid,
40
- body: { 'something' => 'simple' })
41
- end
42
-
43
- Benchmark.ips do |b|
44
- b.report("event_store.sink") do
45
- event_store.sink(new_event)
46
- end
47
- end
@@ -1,181 +0,0 @@
1
- # Demonstrates that sequence IDs may not be inserted linearly with concurrent
2
- # writers.
3
- #
4
- # This script writes events in parallel from a number of forked processes,
5
- # writing events in a continious loop until the program is interrupted.
6
- # The parent process detects gaps in sequence IDs by selecting the last 2
7
- # events based on sequence ID. A gap is detected when the 2 IDs returned from
8
- # that query aren't sequential. The script will proceed to execute 2 subsequent
9
- # queries to see if they show up in the time it takes to complete those before
10
- # moving on.
11
- #
12
- # An easier way to demonstrate this is by using 2 psql consoles:
13
- #
14
- # - Simulate a transaction taking a long time to commit:
15
- # ```
16
- # begin;
17
- # insert into events (..) values (..);
18
- # ```
19
- # - Then, in another console:
20
- # ```
21
- # insert into events (..) values (..);
22
- # select * from events;
23
- # ```
24
- #
25
- # The result is that event sequence ID 2 is visible, but only when the first
26
- # transaction commits is event sequence ID 1 visible.
27
- #
28
- # Why does this happen?
29
- #
30
- # Sequences in Postgres (and most other DBs) are not transactional, changes
31
- # to the sequence are visible to other transactions immediately. Also, inserts
32
- # from the forked writers may be executed in parallel by postgres.
33
- #
34
- # The process of inserting into a table that has a sequence or serial column is
35
- # to first get the next sequence ID (changing global state), then perform the
36
- # insert statement and later commit. In between these 2 steps the sequence ID
37
- # is taken but not visible in the table until the insert statement is
38
- # committed. Gaps in sequence IDs occur when a process takes a sequence ID and
39
- # commits it while another process is in between those 2 steps.
40
- #
41
- # This means another transaction could have taken the next sequence
42
- # ID and committed before that one commits, resulting in a gap in sequence ID's
43
- # when reading.
44
- #
45
- # Why is this a problem?
46
- #
47
- # Event stream processors use the sequence ID to keep track of where they're up to
48
- # in the events table. If a projector processes an event with sequence ID n, it
49
- # assumes that the next event it needs to process will have a sequence ID > n.
50
- # This approach isn't reliable when sequence IDs appear non-linearly, making it
51
- # possible for event stream processors to skip over events.
52
- #
53
- # How does EventSourcery deal with this?
54
- #
55
- # EventSourcery uses n transaction level advisory lock to synchronise inserts
56
- # to the events table within the writeEvents function. Alternatives:
57
- #
58
- # - Write events from 1 process only (serialize at the application level)
59
- # - Detect gaps when reading events and allow time for in-flight transactions
60
- # (the gaps) to commit.
61
- # - Built in eventual consistency. Selects would be restricted to events older
62
- # than 500ms-1s or the transaction timeout to give enough time for in-flight
63
- # transactions to commit.
64
- # - Only query events when catching up, after that rely on events to be
65
- # delivered through the pub/sub mechanism. Given events would be received out
66
- # of order under concurrent writes there's potential for processors to process
67
- # a given event twice if they shutdown after processing a sequence that was
68
- # part of a gap.
69
- #
70
- # Usage:
71
- #
72
- # ❯ bundle exec ruby script/demonstrate_event_sequence_id_gaps.rb
73
- # 89847: starting to write events89846: starting to write events
74
-
75
- # 89848: starting to write events
76
- # 89849: starting to write events
77
- # 89850: starting to write events
78
- # GAP: 1 missing sequence IDs. 78 != 76 + 1. Missing events showed up after 1 subsequent query. IDs: [77]
79
- # GAP: 1 missing sequence IDs. 168 != 166 + 1. Missing events showed up after 1 subsequent query. IDs: [167]
80
- # GAP: 1 missing sequence IDs. 274 != 272 + 1. Missing events showed up after 1 subsequent query. IDs: [273]
81
- # GAP: 1 missing sequence IDs. 341 != 339 + 1. Missing events showed up after 1 subsequent query. IDs: [340]
82
- # GAP: 1 missing sequence IDs. 461 != 459 + 1. Missing events showed up after 1 subsequent query. IDs: [460]
83
- # GAP: 1 missing sequence IDs. 493 != 491 + 1. Missing events showed up after 1 subsequent query. IDs: [492]
84
- # GAP: 2 missing sequence IDs. 621 != 618 + 1. Missing events showed up after 1 subsequent query. IDs: [619, 620]
85
-
86
- require 'sequel'
87
- require 'securerandom'
88
- require 'event_sourcery/postgres'
89
-
90
- def connect
91
- pg_uri = ENV.fetch('BOXEN_POSTGRESQL_URL') { 'postgres://127.0.0.1:5432/' }.dup
92
- pg_uri << 'event_sourcery_test'
93
- Sequel.connect(pg_uri)
94
- end
95
-
96
- EventSourcery.logger.level = :info
97
-
98
- def new_event
99
- EventSourcery::Event.new(type: :item_added,
100
- aggregate_id: SecureRandom.uuid,
101
- body: { 'something' => 'simple' })
102
- end
103
-
104
- def create_events_schema(db)
105
- db.execute 'drop table if exists events'
106
- db.execute 'drop table if exists aggregates'
107
- EventSourcery::Postgres::Schema.create_event_store(db: db)
108
- end
109
-
110
- db = connect
111
- create_events_schema(db)
112
- db.disconnect
113
- sleep 0.3
114
-
115
- NUM_WRITER_PROCESSES = 5
116
- NUM_WRITER_PROCESSES.times do
117
- fork do |pid|
118
- stop = false
119
- Signal.trap(:INT) { stop = true }
120
- db = connect
121
- # when lock_table is set to true an advisory lock is used to synchronise
122
- # inserts and no gaps are detected
123
- event_store = EventSourcery::Postgres::EventStore.new(db, lock_table: false)
124
- puts "#{Process.pid}: starting to write events"
125
- until stop
126
- event_store.sink(new_event)
127
- end
128
- end
129
- end
130
-
131
- stop = false
132
- Signal.trap(:INT) { stop = true }
133
-
134
- def wait_for_missing_ids(db, first_sequence, last_sequence, attempt: 1)
135
- missing_ids = db[:events].where(Sequel.lit("id > ? AND id < ?", first_sequence, last_sequence)).order(:id).map {|e| e[:id] }
136
- expected_missing_ids = (first_sequence+1)..(last_sequence-1)
137
- if missing_ids == expected_missing_ids.to_a
138
- print "Missing events showed up after #{attempt} subsequent query. IDs: #{missing_ids}"
139
- else
140
- if attempt < 2
141
- wait_for_missing_ids(db, first_sequence, last_sequence, attempt: attempt + 1)
142
- else
143
- print "Missing events didn't show up after #{attempt} subsequent queries"
144
- end
145
- end
146
- end
147
-
148
- until stop
149
-
150
- # query for the last 2 sequences in the events table
151
- first_sequence, last_sequence = *db[:events].
152
- order(Sequel.desc(:id)).
153
- select(:id).
154
- limit(2).
155
- map { |e| e[:id] }.
156
- reverse
157
-
158
- next if first_sequence.nil? || last_sequence.nil?
159
-
160
- if last_sequence != first_sequence + 1
161
- num_missing = last_sequence - first_sequence - 1
162
- print "GAP: #{num_missing} missing sequence IDs. #{last_sequence} != #{first_sequence} + 1. "
163
- wait_for_missing_ids(db, first_sequence, last_sequence)
164
- puts
165
- end
166
- end
167
-
168
- Process.waitall
169
-
170
- puts
171
- puts "Looking for gaps in sequence IDs in events table:"
172
- ids = db[:events].select(:id).order(:id).all.map { |e| e[:id] }
173
- expected_ids = (ids.min..ids.max).to_a
174
- missing_ids = (expected_ids - ids)
175
- if missing_ids.empty?
176
- puts "No remaining gaps"
177
- else
178
- missing_ids.each do |id|
179
- puts "Unable to find row with sequence ID #{id}"
180
- end
181
- end