event_sourcery-postgres 0.3.0 → 0.4.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 75bff88c421cb623d91957ca0afec6054a835f2a
4
- data.tar.gz: dd7c0cd3eaf82c26bb328fbbcdd80e810a38b45b
3
+ metadata.gz: e45ae5ad105bd165a2fb4b59be6032488ec925df
4
+ data.tar.gz: c36491b6e980803520d2b3a900207500f694b500
5
5
  SHA512:
6
- metadata.gz: 016f2f36b96fef639e26584458ec2798649d3d81f54ce39a398fb4eec5645a31c278d680134630239cf3efba9ee76cccca5117dd98d56b59d8a8a6ce5b6c2efc
7
- data.tar.gz: c033c7b72cb9ea7c31de75aca7c51c168397ea1eb18703eae709a2262a4f9a84893b740195acb5023fab05433fdaa9db26780232ef4ec416d579d3795cc00884
6
+ metadata.gz: a86c88d7db9e0368d3ddbea1b6b7ce869d8f3465dd1bce3990f0038edffcb5712da10d4182fd0ce46427b2e3aa08b67b7da6712d7aec0abd16f766741a492556
7
+ data.tar.gz: 3f286cd47b3eb36e3b3e65929d6df0d51a1bc8762f1256d1b21c6f8041861fc8087bde5978989c7abb58d6f274add9dca368ca992105a12e5e7af6069e390dc7
data/CHANGELOG.md CHANGED
@@ -6,6 +6,19 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
6
6
 
7
7
  ## [Unreleased]
8
8
 
9
+ ## [0.4.0] - 2017-6-21
10
+ ### Changed
11
+ - Reactors store the UUID of the event being processed in the `causation_id`
12
+ of any emitted events. This replaces the old behaviour of storing id of the
13
+ event being processed in a `_driven_by_event_id` attribute in the emitted
14
+ event's body.
15
+
16
+ ### Added
17
+ - Reactors store the correlation id of the event being processed in the
18
+ `correlation_id` of any emitted events.
19
+ - Added index on the `events` table for `correlation_id` and `causation_id`
20
+ columns.
21
+
9
22
  ## [0.3.0] - 2017-6-16
10
23
  ### Changed
11
24
  - The event store persists the event `correlation_id` and `causation_id`.
@@ -4,30 +4,30 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
  require 'event_sourcery/postgres/version'
5
5
 
6
6
  Gem::Specification.new do |spec|
7
- spec.name = "event_sourcery-postgres"
7
+ spec.name = 'event_sourcery-postgres'
8
8
  spec.version = EventSourcery::Postgres::VERSION
9
- # TODO: update authors
10
- spec.authors = ["Steve Hodgkiss"]
11
- spec.email = ["steve@hodgkiss.me"]
12
9
 
13
- spec.summary = %q{Postgres event store for use with EventSourcery}
14
- spec.homepage = "https://github.com/envato/event_sourcery-postgres"
10
+ spec.authors = ['Envato']
11
+ spec.email = ['rubygems@envato.com']
12
+
13
+ spec.summary = 'Postgres event store for use with EventSourcery'
14
+ spec.homepage = 'https://github.com/envato/event_sourcery-postgres'
15
15
 
16
16
  spec.files = `git ls-files -z`.split("\x0").reject do |f|
17
- f.match(%r{^(test|spec|features)/})
17
+ f.match(%r{^(\.|bin/|Gemfile|Rakefile|script/|spec/)})
18
18
  end
19
- spec.bindir = "exe"
19
+ spec.bindir = 'exe'
20
20
  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
21
- spec.require_paths = ["lib"]
21
+ spec.require_paths = ['lib']
22
22
 
23
23
  spec.required_ruby_version = '>= 2.2.0'
24
24
 
25
25
  spec.add_dependency 'sequel', '~> 4.38'
26
26
  spec.add_dependency 'pg'
27
- spec.add_dependency 'event_sourcery', '>= 0.10.0'
28
- spec.add_development_dependency "bundler", "~> 1.10"
29
- spec.add_development_dependency "rake", "~> 10.0"
30
- spec.add_development_dependency "rspec", "~> 3.0"
31
- spec.add_development_dependency "pry"
32
- spec.add_development_dependency "benchmark-ips"
27
+ spec.add_dependency 'event_sourcery', '>= 0.14.0'
28
+ spec.add_development_dependency 'bundler', '~> 1.10'
29
+ spec.add_development_dependency 'rake', '~> 10.0'
30
+ spec.add_development_dependency 'rspec', '~> 3.0'
31
+ spec.add_development_dependency 'pry'
32
+ spec.add_development_dependency 'benchmark-ips'
33
33
  end
@@ -36,19 +36,13 @@ module EventSourcery
36
36
  order(:id).
37
37
  where(Sequel.lit('id >= ?', id)).
38
38
  limit(limit)
39
- if event_types
40
- query = query.where(type: event_types)
41
- end
42
- query.map do |event_row|
43
- build_event(event_row)
44
- end
39
+ query = query.where(type: event_types) if event_types
40
+ query.map { |event_row| build_event(event_row) }
45
41
  end
46
42
 
47
43
  def latest_event_id(event_types: nil)
48
44
  latest_event = events_table
49
- if event_types
50
- latest_event = latest_event.where(type: event_types)
51
- end
45
+ latest_event = latest_event.where(type: event_types) if event_types
52
46
  latest_event = latest_event.order(:id).last
53
47
  if latest_event
54
48
  latest_event[:id]
@@ -74,9 +68,7 @@ module EventSourcery
74
68
  subscription_master: subscription_master,
75
69
  on_new_events: block
76
70
  }
77
- EventSourcery::EventStore::Subscription.new(args).tap do |s|
78
- s.start
79
- end
71
+ EventSourcery::EventStore::Subscription.new(args).tap(&:start)
80
72
  end
81
73
 
82
74
  private
@@ -113,7 +105,7 @@ module EventSourcery
113
105
 
114
106
  def sql_literal_array(events, type, &block)
115
107
  sql_array = events.map do |event|
116
- to_sql_literal(block.call(event))
108
+ to_sql_literal(block.call(event))
117
109
  end.join(', ')
118
110
  "array[#{sql_array}]::#{type}[]"
119
111
  end
@@ -125,12 +117,12 @@ module EventSourcery
125
117
  def to_sql_literal(value)
126
118
  return 'null' unless value
127
119
  wrapped_value = if Time === value
128
- value.iso8601(6)
129
- elsif Hash === value
130
- Sequel.pg_json(value)
131
- else
132
- value
133
- end
120
+ value.iso8601(6)
121
+ elsif Hash === value
122
+ Sequel.pg_json(value)
123
+ else
124
+ value
125
+ end
134
126
  @pg_connection.literal(wrapped_value)
135
127
  end
136
128
 
@@ -4,7 +4,7 @@ module EventSourcery
4
4
  class OptimisedEventPollWaiter
5
5
  ListenThreadDied = Class.new(StandardError)
6
6
 
7
- def initialize(pg_connection:, timeout: 30, after_listen: proc { })
7
+ def initialize(pg_connection:, timeout: 30, after_listen: proc {})
8
8
  @pg_connection = pg_connection
9
9
  @timeout = timeout
10
10
  @events_queue = QueueWithIntervalCallback.new
@@ -17,7 +17,7 @@ module EventSourcery
17
17
  block.call
18
18
  end
19
19
  start_async(after_listen: after_listen)
20
- catch(:stop) {
20
+ catch(:stop) do
21
21
  block.call
22
22
  loop do
23
23
  ensure_listen_thread_alive!
@@ -25,7 +25,7 @@ module EventSourcery
25
25
  clear_new_event_queue
26
26
  block.call
27
27
  end
28
- }
28
+ end
29
29
  ensure
30
30
  shutdown!
31
31
  end
@@ -33,15 +33,11 @@ module EventSourcery
33
33
  private
34
34
 
35
35
  def shutdown!
36
- if @listen_thread.alive?
37
- @listen_thread.kill
38
- end
36
+ @listen_thread.kill if @listen_thread.alive?
39
37
  end
40
38
 
41
39
  def ensure_listen_thread_alive!
42
- if !@listen_thread.alive?
43
- raise ListenThreadDied
44
- end
40
+ raise ListenThreadDied unless @listen_thread.alive?
45
41
  end
46
42
 
47
43
  def wait_for_new_event_to_appear
@@ -54,16 +50,18 @@ module EventSourcery
54
50
 
55
51
  def start_async(after_listen: nil)
56
52
  after_listen_callback = if after_listen
57
- proc {
53
+ proc do
58
54
  after_listen.call
59
55
  @after_listen.call if @after_listen
60
- }
56
+ end
61
57
  else
62
58
  @after_listen
63
59
  end
64
- @listen_thread = Thread.new { listen_for_new_events(loop: true,
65
- after_listen: after_listen_callback,
66
- timeout: @timeout) }
60
+ @listen_thread = Thread.new do
61
+ listen_for_new_events(loop: true,
62
+ after_listen: after_listen_callback,
63
+ timeout: @timeout)
64
+ end
67
65
  end
68
66
 
69
67
  def listen_for_new_events(loop: true, after_listen: nil, timeout: 30)
@@ -71,9 +69,7 @@ module EventSourcery
71
69
  loop: loop,
72
70
  after_listen: after_listen,
73
71
  timeout: timeout) do |_channel, _pid, _payload|
74
- if @events_queue.empty?
75
- @events_queue.push(:new_event_arrived)
76
- end
72
+ @events_queue.push(:new_event_arrived) if @events_queue.empty?
77
73
  end
78
74
  end
79
75
  end
@@ -6,12 +6,12 @@ module EventSourcery
6
6
  base.prepend(TableOwner)
7
7
  base.include(InstanceMethods)
8
8
  base.class_eval do
9
- alias project process
9
+ alias_method :project, :process
10
10
 
11
11
  class << self
12
- alias project process
13
- alias projects_events processes_events
14
- alias projector_name processor_name
12
+ alias_method :project, :process
13
+ alias_method :projects_events, :processes_events
14
+ alias_method :projector_name, :processor_name
15
15
  end
16
16
  end
17
17
  end
@@ -3,7 +3,7 @@ module EventSourcery
3
3
  class QueueWithIntervalCallback < ::Queue
4
4
  attr_accessor :callback
5
5
 
6
- def initialize(callback: proc { }, callback_interval: EventSourcery::Postgres.config.callback_interval_if_no_new_events, poll_interval: 0.1)
6
+ def initialize(callback: proc {}, callback_interval: EventSourcery::Postgres.config.callback_interval_if_no_new_events, poll_interval: 0.1)
7
7
  @callback = callback
8
8
  @callback_interval = callback_interval
9
9
  @poll_interval = poll_interval
@@ -20,7 +20,7 @@ module EventSourcery
20
20
  def pop_with_interval_callback
21
21
  time = Time.now
22
22
  loop do
23
- return pop(true) if !empty?
23
+ return pop(true) unless empty?
24
24
  if @callback_interval && Time.now > time + @callback_interval
25
25
  @callback.call
26
26
  time = Time.now
@@ -45,20 +45,18 @@ module EventSourcery
45
45
  end
46
46
  end
47
47
 
48
- DRIVEN_BY_EVENT_PAYLOAD_KEY = :_driven_by_event_id
49
-
50
48
  private
51
49
 
52
50
  attr_reader :event_sink, :event_source
53
51
 
54
52
  def emit_event(event_or_hash, &block)
55
53
  event = if Event === event_or_hash
56
- event_or_hash
57
- else
58
- Event.new(event_or_hash)
59
- end
54
+ event_or_hash
55
+ else
56
+ Event.new(event_or_hash)
57
+ end
60
58
  raise UndeclaredEventEmissionError unless self.class.emits_event?(event.class)
61
- event.body.merge!(DRIVEN_BY_EVENT_PAYLOAD_KEY => _event.id)
59
+ event = event.with(causation_id: _event.uuid, correlation_id: _event.correlation_id)
62
60
  invoke_action_and_emit_event(event, block)
63
61
  EventSourcery.logger.debug { "[#{self.processor_name}] Emitted event: #{event.inspect}" }
64
62
  end
@@ -1,7 +1,7 @@
1
1
  module EventSourcery
2
2
  module Postgres
3
3
  module Schema
4
- extend self
4
+ module_function
5
5
 
6
6
  def create_event_store(db: EventSourcery::Postgres.config.event_store_database,
7
7
  events_table_name: EventSourcery::Postgres.config.events_table_name,
@@ -28,6 +28,8 @@ module EventSourcery
28
28
  index [:aggregate_id, :version], unique: true
29
29
  index :uuid, unique: true
30
30
  index :type
31
+ index :correlation_id
32
+ index :causation_id
31
33
  index :created_at
32
34
  end
33
35
  end
@@ -67,7 +67,7 @@ module EventSourcery
67
67
  end
68
68
 
69
69
  def table_name_prefixed(name)
70
- [table_prefix, name].compact.join("_").to_sym
70
+ [table_prefix, name].compact.join('_').to_sym
71
71
  end
72
72
  end
73
73
  end
@@ -5,7 +5,7 @@ module EventSourcery
5
5
  table_name: EventSourcery::Postgres.config.tracker_table_name,
6
6
  obtain_processor_lock: true)
7
7
  @connection = connection
8
- @table_name = table_name
8
+ @table_name = table_name.to_sym
9
9
  @obtain_processor_lock = obtain_processor_lock
10
10
  end
11
11
 
@@ -13,7 +13,7 @@ module EventSourcery
13
13
  create_table_if_not_exists if EventSourcery::Postgres.config.auto_create_projector_tracker
14
14
 
15
15
  unless tracker_table_exists?
16
- raise UnableToLockProcessorError, "Projector tracker table does not exist"
16
+ raise UnableToLockProcessorError, 'Projector tracker table does not exist'
17
17
  end
18
18
 
19
19
  if processor_name
@@ -27,7 +27,7 @@ module EventSourcery
27
27
  def processed_event(processor_name, event_id)
28
28
  table.
29
29
  where(name: processor_name.to_s).
30
- update(last_processed_event_id: event_id)
30
+ update(last_processed_event_id: event_id)
31
31
  true
32
32
  end
33
33
 
@@ -44,9 +44,7 @@ module EventSourcery
44
44
 
45
45
  def last_processed_event_id(processor_name)
46
46
  track_entry = table.where(name: processor_name.to_s).first
47
- if track_entry
48
- track_entry[:last_processed_event_id]
49
- end
47
+ track_entry[:last_processed_event_id] if track_entry
50
48
  end
51
49
 
52
50
  def tracked_processors
@@ -1,5 +1,5 @@
1
1
  module EventSourcery
2
2
  module Postgres
3
- VERSION = "0.3.0"
3
+ VERSION = '0.4.0'.freeze
4
4
  end
5
5
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: event_sourcery-postgres
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.0
4
+ version: 0.4.0
5
5
  platform: ruby
6
6
  authors:
7
- - Steve Hodgkiss
7
+ - Envato
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2017-06-16 00:00:00.000000000 Z
11
+ date: 2017-06-21 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: sequel
@@ -44,14 +44,14 @@ dependencies:
44
44
  requirements:
45
45
  - - ">="
46
46
  - !ruby/object:Gem::Version
47
- version: 0.10.0
47
+ version: 0.14.0
48
48
  type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
52
  - - ">="
53
53
  - !ruby/object:Gem::Version
54
- version: 0.10.0
54
+ version: 0.14.0
55
55
  - !ruby/object:Gem::Dependency
56
56
  name: bundler
57
57
  requirement: !ruby/object:Gem::Requirement
@@ -124,22 +124,15 @@ dependencies:
124
124
  version: '0'
125
125
  description:
126
126
  email:
127
- - steve@hodgkiss.me
127
+ - rubygems@envato.com
128
128
  executables: []
129
129
  extensions: []
130
130
  extra_rdoc_files: []
131
131
  files:
132
- - ".gitignore"
133
- - ".rspec"
134
- - ".travis.yml"
135
132
  - CHANGELOG.md
136
133
  - CODE_OF_CONDUCT.md
137
- - Gemfile
138
134
  - LICENSE.txt
139
135
  - README.md
140
- - Rakefile
141
- - bin/console
142
- - bin/setup
143
136
  - event_sourcery-postgres.gemspec
144
137
  - lib/event_sourcery/postgres.rb
145
138
  - lib/event_sourcery/postgres/config.rb
@@ -152,9 +145,6 @@ files:
152
145
  - lib/event_sourcery/postgres/table_owner.rb
153
146
  - lib/event_sourcery/postgres/tracker.rb
154
147
  - lib/event_sourcery/postgres/version.rb
155
- - script/bench_reading_events.rb
156
- - script/bench_writing_events.rb
157
- - script/demonstrate_event_sequence_id_gaps.rb
158
148
  homepage: https://github.com/envato/event_sourcery-postgres
159
149
  licenses: []
160
150
  metadata: {}
data/.gitignore DELETED
@@ -1,13 +0,0 @@
1
- /.bundle/
2
- /.yardoc
3
- /Gemfile.lock
4
- /_yardoc/
5
- /coverage/
6
- /doc/
7
- /pkg/
8
- /spec/reports/
9
- /tmp/
10
- /vendor/
11
-
12
- # rspec failure tracking
13
- .rspec_status
data/.rspec DELETED
@@ -1,3 +0,0 @@
1
- --require spec_helper
2
- --format documentation
3
- --color
data/.travis.yml DELETED
@@ -1,12 +0,0 @@
1
- sudo: false
2
- language: ruby
3
- rvm:
4
- - 2.2
5
- - 2.3
6
- - 2.4
7
- before_install:
8
- - gem install bundler
9
- before_script:
10
- - psql -c 'create database event_sourcery_test;' -U postgres
11
- addons:
12
- postgresql: 9.4
data/Gemfile DELETED
@@ -1,5 +0,0 @@
1
- source 'https://rubygems.org'
2
-
3
- ruby '>= 2.2.0'
4
-
5
- gemspec
data/Rakefile DELETED
@@ -1,6 +0,0 @@
1
- require "bundler/gem_tasks"
2
- require "rspec/core/rake_task"
3
-
4
- RSpec::Core::RakeTask.new(:spec)
5
-
6
- task :default => :spec
data/bin/console DELETED
@@ -1,14 +0,0 @@
1
- #!/usr/bin/env ruby
2
-
3
- require "bundler/setup"
4
- require "event_sourcery/postgres"
5
-
6
- # You can add fixtures and/or initialization code here to make experimenting
7
- # with your gem easier. You can also use a different console, if you like.
8
-
9
- # (If you use this, don't forget to add pry to your Gemfile!)
10
- # require "pry"
11
- # Pry.start
12
-
13
- require "irb"
14
- IRB.start(__FILE__)
data/bin/setup DELETED
@@ -1,15 +0,0 @@
1
- #!/usr/bin/env bash
2
- set -euo pipefail
3
- IFS=$'\n\t'
4
- set -vx
5
-
6
- bundle install
7
-
8
- echo
9
- echo "--- Preparing test databases"
10
- echo
11
-
12
- dropdb event_sourcery_test || echo 0
13
- createdb event_sourcery_test
14
-
15
- # Do any other automated setup that you need to do here
@@ -1,63 +0,0 @@
1
- # Usage:
2
- #
3
- # ❯ bundle exec ruby script/bench_reading_events.rb
4
- # Creating 10000 events
5
- # Took 42.35533199999918 to create events
6
- # Took 4.9821800000027 to read all events
7
- # ^ results from running on a 2016 MacBook
8
-
9
- require 'benchmark'
10
- require 'securerandom'
11
- require 'sequel'
12
- require 'event_sourcery/postgres'
13
-
14
- pg_uri = ENV.fetch('BOXEN_POSTGRESQL_URL') { 'postgres://127.0.0.1:5432/' }.dup
15
- pg_uri << 'event_sourcery_test'
16
- pg_connection = Sequel.connect(pg_uri)
17
-
18
- EventSourcery.configure do |config|
19
- config.postgres.event_store_database = pg_connection
20
- config.postgres.projections_database = pg_connection
21
- config.logger.level = :fatal
22
- end
23
-
24
- def create_events_schema(pg_connection)
25
- pg_connection.execute 'drop table if exists events'
26
- pg_connection.execute 'drop table if exists aggregates'
27
- EventSourcery::Postgres::Schema.create_event_store(db: pg_connection)
28
- end
29
-
30
- event_store = EventSourcery::Postgres.config.event_store
31
-
32
- EVENT_TYPES = %i[
33
- item_added
34
- item_removed
35
- item_starred
36
- ]
37
-
38
- def new_event(uuid)
39
- EventSourcery::Event.new(type: EVENT_TYPES.sample,
40
- aggregate_id: uuid,
41
- body: { 'something' => 'simple' })
42
- end
43
-
44
- create_events_schema(pg_connection)
45
-
46
- NUM_EVENTS = 10_000
47
- puts "Creating #{NUM_EVENTS} events"
48
- time = Benchmark.realtime do
49
- uuid = SecureRandom.uuid
50
- NUM_EVENTS.times do
51
- event_store.sink(new_event(uuid))
52
- end
53
- end
54
- puts "Took #{time} to create events"
55
-
56
- seen_events_count = 0
57
- time = Benchmark.realtime do
58
- event_store.subscribe(from_id: 0, subscription_master: EventSourcery::EventStore::SignalHandlingSubscriptionMaster.new) do |events|
59
- seen_events_count += events.count
60
- throw :stop if seen_events_count >= NUM_EVENTS
61
- end
62
- end
63
- puts "Took #{time} to read all events"
@@ -1,47 +0,0 @@
1
- # Usage:
2
- #
3
- # ❯ bundle exec ruby script/bench_writing_events.rb
4
- # Warming up --------------------------------------
5
- # event_store.sink
6
- # 70.000 i/100ms
7
- # Calculating -------------------------------------
8
- # event_store.sink
9
- # 522.007 (±10.9%) i/s - 2.590k in 5.021909s
10
- #
11
- # ^ results from running on a 2016 MacBook
12
-
13
- require 'benchmark/ips'
14
- require 'securerandom'
15
- require 'sequel'
16
- require 'event_sourcery/postgres'
17
-
18
- pg_uri = ENV.fetch('BOXEN_POSTGRESQL_URL') { 'postgres://127.0.0.1:5432/' }.dup
19
- pg_uri << 'event_sourcery_test'
20
- pg_connection = Sequel.connect(pg_uri)
21
-
22
- EventSourcery.configure do |config|
23
- config.postgres.event_store_database = pg_connection
24
- config.postgres.projections_database = pg_connection
25
- config.logger.level = :fatal
26
- end
27
-
28
- def create_schema(pg_connection)
29
- pg_connection.execute 'drop table if exists events'
30
- pg_connection.execute 'drop table if exists aggregates'
31
- EventSourcery::Postgres::Schema.create_event_store(db: pg_connection)
32
- end
33
-
34
- create_schema(pg_connection)
35
- event_store = EventSourcery::Postgres::EventStore.new(pg_connection)
36
-
37
- def new_event
38
- EventSourcery::Event.new(type: :item_added,
39
- aggregate_id: SecureRandom.uuid,
40
- body: { 'something' => 'simple' })
41
- end
42
-
43
- Benchmark.ips do |b|
44
- b.report("event_store.sink") do
45
- event_store.sink(new_event)
46
- end
47
- end
@@ -1,181 +0,0 @@
1
- # Demonstrates that sequence IDs may not be inserted linearly with concurrent
2
- # writers.
3
- #
4
- # This script writes events in parallel from a number of forked processes,
5
- # writing events in a continious loop until the program is interrupted.
6
- # The parent process detects gaps in sequence IDs by selecting the last 2
7
- # events based on sequence ID. A gap is detected when the 2 IDs returned from
8
- # that query aren't sequential. The script will proceed to execute 2 subsequent
9
- # queries to see if they show up in the time it takes to complete those before
10
- # moving on.
11
- #
12
- # An easier way to demonstrate this is by using 2 psql consoles:
13
- #
14
- # - Simulate a transaction taking a long time to commit:
15
- # ```
16
- # begin;
17
- # insert into events (..) values (..);
18
- # ```
19
- # - Then, in another console:
20
- # ```
21
- # insert into events (..) values (..);
22
- # select * from events;
23
- # ```
24
- #
25
- # The result is that event sequence ID 2 is visible, but only when the first
26
- # transaction commits is event sequence ID 1 visible.
27
- #
28
- # Why does this happen?
29
- #
30
- # Sequences in Postgres (and most other DBs) are not transactional, changes
31
- # to the sequence are visible to other transactions immediately. Also, inserts
32
- # from the forked writers may be executed in parallel by postgres.
33
- #
34
- # The process of inserting into a table that has a sequence or serial column is
35
- # to first get the next sequence ID (changing global state), then perform the
36
- # insert statement and later commit. In between these 2 steps the sequence ID
37
- # is taken but not visible in the table until the insert statement is
38
- # committed. Gaps in sequence IDs occur when a process takes a sequence ID and
39
- # commits it while another process is in between those 2 steps.
40
- #
41
- # This means another transaction could have taken the next sequence
42
- # ID and committed before that one commits, resulting in a gap in sequence ID's
43
- # when reading.
44
- #
45
- # Why is this a problem?
46
- #
47
- # Event stream processors use the sequence ID to keep track of where they're up to
48
- # in the events table. If a projector processes an event with sequence ID n, it
49
- # assumes that the next event it needs to process will have a sequence ID > n.
50
- # This approach isn't reliable when sequence IDs appear non-linearly, making it
51
- # possible for event stream processors to skip over events.
52
- #
53
- # How does EventSourcery deal with this?
54
- #
55
- # EventSourcery uses n transaction level advisory lock to synchronise inserts
56
- # to the events table within the writeEvents function. Alternatives:
57
- #
58
- # - Write events from 1 process only (serialize at the application level)
59
- # - Detect gaps when reading events and allow time for in-flight transactions
60
- # (the gaps) to commit.
61
- # - Built in eventual consistency. Selects would be restricted to events older
62
- # than 500ms-1s or the transaction timeout to give enough time for in-flight
63
- # transactions to commit.
64
- # - Only query events when catching up, after that rely on events to be
65
- # delivered through the pub/sub mechanism. Given events would be received out
66
- # of order under concurrent writes there's potential for processors to process
67
- # a given event twice if they shutdown after processing a sequence that was
68
- # part of a gap.
69
- #
70
- # Usage:
71
- #
72
- # ❯ bundle exec ruby script/demonstrate_event_sequence_id_gaps.rb
73
- # 89847: starting to write events89846: starting to write events
74
-
75
- # 89848: starting to write events
76
- # 89849: starting to write events
77
- # 89850: starting to write events
78
- # GAP: 1 missing sequence IDs. 78 != 76 + 1. Missing events showed up after 1 subsequent query. IDs: [77]
79
- # GAP: 1 missing sequence IDs. 168 != 166 + 1. Missing events showed up after 1 subsequent query. IDs: [167]
80
- # GAP: 1 missing sequence IDs. 274 != 272 + 1. Missing events showed up after 1 subsequent query. IDs: [273]
81
- # GAP: 1 missing sequence IDs. 341 != 339 + 1. Missing events showed up after 1 subsequent query. IDs: [340]
82
- # GAP: 1 missing sequence IDs. 461 != 459 + 1. Missing events showed up after 1 subsequent query. IDs: [460]
83
- # GAP: 1 missing sequence IDs. 493 != 491 + 1. Missing events showed up after 1 subsequent query. IDs: [492]
84
- # GAP: 2 missing sequence IDs. 621 != 618 + 1. Missing events showed up after 1 subsequent query. IDs: [619, 620]
85
-
86
- require 'sequel'
87
- require 'securerandom'
88
- require 'event_sourcery/postgres'
89
-
90
- def connect
91
- pg_uri = ENV.fetch('BOXEN_POSTGRESQL_URL') { 'postgres://127.0.0.1:5432/' }.dup
92
- pg_uri << 'event_sourcery_test'
93
- Sequel.connect(pg_uri)
94
- end
95
-
96
- EventSourcery.logger.level = :info
97
-
98
- def new_event
99
- EventSourcery::Event.new(type: :item_added,
100
- aggregate_id: SecureRandom.uuid,
101
- body: { 'something' => 'simple' })
102
- end
103
-
104
- def create_events_schema(db)
105
- db.execute 'drop table if exists events'
106
- db.execute 'drop table if exists aggregates'
107
- EventSourcery::Postgres::Schema.create_event_store(db: db)
108
- end
109
-
110
- db = connect
111
- create_events_schema(db)
112
- db.disconnect
113
- sleep 0.3
114
-
115
- NUM_WRITER_PROCESSES = 5
116
- NUM_WRITER_PROCESSES.times do
117
- fork do |pid|
118
- stop = false
119
- Signal.trap(:INT) { stop = true }
120
- db = connect
121
- # when lock_table is set to true an advisory lock is used to synchronise
122
- # inserts and no gaps are detected
123
- event_store = EventSourcery::Postgres::EventStore.new(db, lock_table: false)
124
- puts "#{Process.pid}: starting to write events"
125
- until stop
126
- event_store.sink(new_event)
127
- end
128
- end
129
- end
130
-
131
- stop = false
132
- Signal.trap(:INT) { stop = true }
133
-
134
- def wait_for_missing_ids(db, first_sequence, last_sequence, attempt: 1)
135
- missing_ids = db[:events].where(Sequel.lit("id > ? AND id < ?", first_sequence, last_sequence)).order(:id).map {|e| e[:id] }
136
- expected_missing_ids = (first_sequence+1)..(last_sequence-1)
137
- if missing_ids == expected_missing_ids.to_a
138
- print "Missing events showed up after #{attempt} subsequent query. IDs: #{missing_ids}"
139
- else
140
- if attempt < 2
141
- wait_for_missing_ids(db, first_sequence, last_sequence, attempt: attempt + 1)
142
- else
143
- print "Missing events didn't show up after #{attempt} subsequent queries"
144
- end
145
- end
146
- end
147
-
148
- until stop
149
-
150
- # query for the last 2 sequences in the events table
151
- first_sequence, last_sequence = *db[:events].
152
- order(Sequel.desc(:id)).
153
- select(:id).
154
- limit(2).
155
- map { |e| e[:id] }.
156
- reverse
157
-
158
- next if first_sequence.nil? || last_sequence.nil?
159
-
160
- if last_sequence != first_sequence + 1
161
- num_missing = last_sequence - first_sequence - 1
162
- print "GAP: #{num_missing} missing sequence IDs. #{last_sequence} != #{first_sequence} + 1. "
163
- wait_for_missing_ids(db, first_sequence, last_sequence)
164
- puts
165
- end
166
- end
167
-
168
- Process.waitall
169
-
170
- puts
171
- puts "Looking for gaps in sequence IDs in events table:"
172
- ids = db[:events].select(:id).order(:id).all.map { |e| e[:id] }
173
- expected_ids = (ids.min..ids.max).to_a
174
- missing_ids = (expected_ids - ids)
175
- if missing_ids.empty?
176
- puts "No remaining gaps"
177
- else
178
- missing_ids.each do |id|
179
- puts "Unable to find row with sequence ID #{id}"
180
- end
181
- end