event_sourcery-postgres 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +13 -0
- data/event_sourcery-postgres.gemspec +15 -15
- data/lib/event_sourcery/postgres/event_store.rb +11 -19
- data/lib/event_sourcery/postgres/optimised_event_poll_waiter.rb +13 -17
- data/lib/event_sourcery/postgres/projector.rb +4 -4
- data/lib/event_sourcery/postgres/queue_with_interval_callback.rb +2 -2
- data/lib/event_sourcery/postgres/reactor.rb +5 -7
- data/lib/event_sourcery/postgres/schema.rb +3 -1
- data/lib/event_sourcery/postgres/table_owner.rb +1 -1
- data/lib/event_sourcery/postgres/tracker.rb +4 -6
- data/lib/event_sourcery/postgres/version.rb +1 -1
- metadata +6 -16
- data/.gitignore +0 -13
- data/.rspec +0 -3
- data/.travis.yml +0 -12
- data/Gemfile +0 -5
- data/Rakefile +0 -6
- data/bin/console +0 -14
- data/bin/setup +0 -15
- data/script/bench_reading_events.rb +0 -63
- data/script/bench_writing_events.rb +0 -47
- data/script/demonstrate_event_sequence_id_gaps.rb +0 -181
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e45ae5ad105bd165a2fb4b59be6032488ec925df
|
4
|
+
data.tar.gz: c36491b6e980803520d2b3a900207500f694b500
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a86c88d7db9e0368d3ddbea1b6b7ce869d8f3465dd1bce3990f0038edffcb5712da10d4182fd0ce46427b2e3aa08b67b7da6712d7aec0abd16f766741a492556
|
7
|
+
data.tar.gz: 3f286cd47b3eb36e3b3e65929d6df0d51a1bc8762f1256d1b21c6f8041861fc8087bde5978989c7abb58d6f274add9dca368ca992105a12e5e7af6069e390dc7
|
data/CHANGELOG.md
CHANGED
@@ -6,6 +6,19 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
|
|
6
6
|
|
7
7
|
## [Unreleased]
|
8
8
|
|
9
|
+
## [0.4.0] - 2017-6-21
|
10
|
+
### Changed
|
11
|
+
- Reactors store the UUID of the event being processed in the `causation_id`
|
12
|
+
of any emitted events. This replaces the old behaviour of storing id of the
|
13
|
+
event being processed in a `_driven_by_event_id` attribute in the emitted
|
14
|
+
event's body.
|
15
|
+
|
16
|
+
### Added
|
17
|
+
- Reactors store the correlation id of the event being processed in the
|
18
|
+
`correlation_id` of any emitted events.
|
19
|
+
- Added index on the `events` table for `correlation_id` and `causation_id`
|
20
|
+
columns.
|
21
|
+
|
9
22
|
## [0.3.0] - 2017-6-16
|
10
23
|
### Changed
|
11
24
|
- The event store persists the event `correlation_id` and `causation_id`.
|
@@ -4,30 +4,30 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
require 'event_sourcery/postgres/version'
|
5
5
|
|
6
6
|
Gem::Specification.new do |spec|
|
7
|
-
spec.name =
|
7
|
+
spec.name = 'event_sourcery-postgres'
|
8
8
|
spec.version = EventSourcery::Postgres::VERSION
|
9
|
-
# TODO: update authors
|
10
|
-
spec.authors = ["Steve Hodgkiss"]
|
11
|
-
spec.email = ["steve@hodgkiss.me"]
|
12
9
|
|
13
|
-
spec.
|
14
|
-
spec.
|
10
|
+
spec.authors = ['Envato']
|
11
|
+
spec.email = ['rubygems@envato.com']
|
12
|
+
|
13
|
+
spec.summary = 'Postgres event store for use with EventSourcery'
|
14
|
+
spec.homepage = 'https://github.com/envato/event_sourcery-postgres'
|
15
15
|
|
16
16
|
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
17
|
-
f.match(%r{^(
|
17
|
+
f.match(%r{^(\.|bin/|Gemfile|Rakefile|script/|spec/)})
|
18
18
|
end
|
19
|
-
spec.bindir =
|
19
|
+
spec.bindir = 'exe'
|
20
20
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
21
|
-
spec.require_paths = [
|
21
|
+
spec.require_paths = ['lib']
|
22
22
|
|
23
23
|
spec.required_ruby_version = '>= 2.2.0'
|
24
24
|
|
25
25
|
spec.add_dependency 'sequel', '~> 4.38'
|
26
26
|
spec.add_dependency 'pg'
|
27
|
-
spec.add_dependency 'event_sourcery', '>= 0.
|
28
|
-
spec.add_development_dependency
|
29
|
-
spec.add_development_dependency
|
30
|
-
spec.add_development_dependency
|
31
|
-
spec.add_development_dependency
|
32
|
-
spec.add_development_dependency
|
27
|
+
spec.add_dependency 'event_sourcery', '>= 0.14.0'
|
28
|
+
spec.add_development_dependency 'bundler', '~> 1.10'
|
29
|
+
spec.add_development_dependency 'rake', '~> 10.0'
|
30
|
+
spec.add_development_dependency 'rspec', '~> 3.0'
|
31
|
+
spec.add_development_dependency 'pry'
|
32
|
+
spec.add_development_dependency 'benchmark-ips'
|
33
33
|
end
|
@@ -36,19 +36,13 @@ module EventSourcery
|
|
36
36
|
order(:id).
|
37
37
|
where(Sequel.lit('id >= ?', id)).
|
38
38
|
limit(limit)
|
39
|
-
if event_types
|
40
|
-
|
41
|
-
end
|
42
|
-
query.map do |event_row|
|
43
|
-
build_event(event_row)
|
44
|
-
end
|
39
|
+
query = query.where(type: event_types) if event_types
|
40
|
+
query.map { |event_row| build_event(event_row) }
|
45
41
|
end
|
46
42
|
|
47
43
|
def latest_event_id(event_types: nil)
|
48
44
|
latest_event = events_table
|
49
|
-
if event_types
|
50
|
-
latest_event = latest_event.where(type: event_types)
|
51
|
-
end
|
45
|
+
latest_event = latest_event.where(type: event_types) if event_types
|
52
46
|
latest_event = latest_event.order(:id).last
|
53
47
|
if latest_event
|
54
48
|
latest_event[:id]
|
@@ -74,9 +68,7 @@ module EventSourcery
|
|
74
68
|
subscription_master: subscription_master,
|
75
69
|
on_new_events: block
|
76
70
|
}
|
77
|
-
EventSourcery::EventStore::Subscription.new(args).tap
|
78
|
-
s.start
|
79
|
-
end
|
71
|
+
EventSourcery::EventStore::Subscription.new(args).tap(&:start)
|
80
72
|
end
|
81
73
|
|
82
74
|
private
|
@@ -113,7 +105,7 @@ module EventSourcery
|
|
113
105
|
|
114
106
|
def sql_literal_array(events, type, &block)
|
115
107
|
sql_array = events.map do |event|
|
116
|
-
|
108
|
+
to_sql_literal(block.call(event))
|
117
109
|
end.join(', ')
|
118
110
|
"array[#{sql_array}]::#{type}[]"
|
119
111
|
end
|
@@ -125,12 +117,12 @@ module EventSourcery
|
|
125
117
|
def to_sql_literal(value)
|
126
118
|
return 'null' unless value
|
127
119
|
wrapped_value = if Time === value
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
120
|
+
value.iso8601(6)
|
121
|
+
elsif Hash === value
|
122
|
+
Sequel.pg_json(value)
|
123
|
+
else
|
124
|
+
value
|
125
|
+
end
|
134
126
|
@pg_connection.literal(wrapped_value)
|
135
127
|
end
|
136
128
|
|
@@ -4,7 +4,7 @@ module EventSourcery
|
|
4
4
|
class OptimisedEventPollWaiter
|
5
5
|
ListenThreadDied = Class.new(StandardError)
|
6
6
|
|
7
|
-
def initialize(pg_connection:, timeout: 30, after_listen: proc {
|
7
|
+
def initialize(pg_connection:, timeout: 30, after_listen: proc {})
|
8
8
|
@pg_connection = pg_connection
|
9
9
|
@timeout = timeout
|
10
10
|
@events_queue = QueueWithIntervalCallback.new
|
@@ -17,7 +17,7 @@ module EventSourcery
|
|
17
17
|
block.call
|
18
18
|
end
|
19
19
|
start_async(after_listen: after_listen)
|
20
|
-
catch(:stop)
|
20
|
+
catch(:stop) do
|
21
21
|
block.call
|
22
22
|
loop do
|
23
23
|
ensure_listen_thread_alive!
|
@@ -25,7 +25,7 @@ module EventSourcery
|
|
25
25
|
clear_new_event_queue
|
26
26
|
block.call
|
27
27
|
end
|
28
|
-
|
28
|
+
end
|
29
29
|
ensure
|
30
30
|
shutdown!
|
31
31
|
end
|
@@ -33,15 +33,11 @@ module EventSourcery
|
|
33
33
|
private
|
34
34
|
|
35
35
|
def shutdown!
|
36
|
-
if @listen_thread.alive?
|
37
|
-
@listen_thread.kill
|
38
|
-
end
|
36
|
+
@listen_thread.kill if @listen_thread.alive?
|
39
37
|
end
|
40
38
|
|
41
39
|
def ensure_listen_thread_alive!
|
42
|
-
|
43
|
-
raise ListenThreadDied
|
44
|
-
end
|
40
|
+
raise ListenThreadDied unless @listen_thread.alive?
|
45
41
|
end
|
46
42
|
|
47
43
|
def wait_for_new_event_to_appear
|
@@ -54,16 +50,18 @@ module EventSourcery
|
|
54
50
|
|
55
51
|
def start_async(after_listen: nil)
|
56
52
|
after_listen_callback = if after_listen
|
57
|
-
proc
|
53
|
+
proc do
|
58
54
|
after_listen.call
|
59
55
|
@after_listen.call if @after_listen
|
60
|
-
|
56
|
+
end
|
61
57
|
else
|
62
58
|
@after_listen
|
63
59
|
end
|
64
|
-
@listen_thread = Thread.new
|
65
|
-
|
66
|
-
|
60
|
+
@listen_thread = Thread.new do
|
61
|
+
listen_for_new_events(loop: true,
|
62
|
+
after_listen: after_listen_callback,
|
63
|
+
timeout: @timeout)
|
64
|
+
end
|
67
65
|
end
|
68
66
|
|
69
67
|
def listen_for_new_events(loop: true, after_listen: nil, timeout: 30)
|
@@ -71,9 +69,7 @@ module EventSourcery
|
|
71
69
|
loop: loop,
|
72
70
|
after_listen: after_listen,
|
73
71
|
timeout: timeout) do |_channel, _pid, _payload|
|
74
|
-
if @events_queue.empty?
|
75
|
-
@events_queue.push(:new_event_arrived)
|
76
|
-
end
|
72
|
+
@events_queue.push(:new_event_arrived) if @events_queue.empty?
|
77
73
|
end
|
78
74
|
end
|
79
75
|
end
|
@@ -6,12 +6,12 @@ module EventSourcery
|
|
6
6
|
base.prepend(TableOwner)
|
7
7
|
base.include(InstanceMethods)
|
8
8
|
base.class_eval do
|
9
|
-
|
9
|
+
alias_method :project, :process
|
10
10
|
|
11
11
|
class << self
|
12
|
-
|
13
|
-
|
14
|
-
|
12
|
+
alias_method :project, :process
|
13
|
+
alias_method :projects_events, :processes_events
|
14
|
+
alias_method :projector_name, :processor_name
|
15
15
|
end
|
16
16
|
end
|
17
17
|
end
|
@@ -3,7 +3,7 @@ module EventSourcery
|
|
3
3
|
class QueueWithIntervalCallback < ::Queue
|
4
4
|
attr_accessor :callback
|
5
5
|
|
6
|
-
def initialize(callback: proc {
|
6
|
+
def initialize(callback: proc {}, callback_interval: EventSourcery::Postgres.config.callback_interval_if_no_new_events, poll_interval: 0.1)
|
7
7
|
@callback = callback
|
8
8
|
@callback_interval = callback_interval
|
9
9
|
@poll_interval = poll_interval
|
@@ -20,7 +20,7 @@ module EventSourcery
|
|
20
20
|
def pop_with_interval_callback
|
21
21
|
time = Time.now
|
22
22
|
loop do
|
23
|
-
return pop(true)
|
23
|
+
return pop(true) unless empty?
|
24
24
|
if @callback_interval && Time.now > time + @callback_interval
|
25
25
|
@callback.call
|
26
26
|
time = Time.now
|
@@ -45,20 +45,18 @@ module EventSourcery
|
|
45
45
|
end
|
46
46
|
end
|
47
47
|
|
48
|
-
DRIVEN_BY_EVENT_PAYLOAD_KEY = :_driven_by_event_id
|
49
|
-
|
50
48
|
private
|
51
49
|
|
52
50
|
attr_reader :event_sink, :event_source
|
53
51
|
|
54
52
|
def emit_event(event_or_hash, &block)
|
55
53
|
event = if Event === event_or_hash
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
54
|
+
event_or_hash
|
55
|
+
else
|
56
|
+
Event.new(event_or_hash)
|
57
|
+
end
|
60
58
|
raise UndeclaredEventEmissionError unless self.class.emits_event?(event.class)
|
61
|
-
event.
|
59
|
+
event = event.with(causation_id: _event.uuid, correlation_id: _event.correlation_id)
|
62
60
|
invoke_action_and_emit_event(event, block)
|
63
61
|
EventSourcery.logger.debug { "[#{self.processor_name}] Emitted event: #{event.inspect}" }
|
64
62
|
end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
module EventSourcery
|
2
2
|
module Postgres
|
3
3
|
module Schema
|
4
|
-
|
4
|
+
module_function
|
5
5
|
|
6
6
|
def create_event_store(db: EventSourcery::Postgres.config.event_store_database,
|
7
7
|
events_table_name: EventSourcery::Postgres.config.events_table_name,
|
@@ -28,6 +28,8 @@ module EventSourcery
|
|
28
28
|
index [:aggregate_id, :version], unique: true
|
29
29
|
index :uuid, unique: true
|
30
30
|
index :type
|
31
|
+
index :correlation_id
|
32
|
+
index :causation_id
|
31
33
|
index :created_at
|
32
34
|
end
|
33
35
|
end
|
@@ -5,7 +5,7 @@ module EventSourcery
|
|
5
5
|
table_name: EventSourcery::Postgres.config.tracker_table_name,
|
6
6
|
obtain_processor_lock: true)
|
7
7
|
@connection = connection
|
8
|
-
@table_name = table_name
|
8
|
+
@table_name = table_name.to_sym
|
9
9
|
@obtain_processor_lock = obtain_processor_lock
|
10
10
|
end
|
11
11
|
|
@@ -13,7 +13,7 @@ module EventSourcery
|
|
13
13
|
create_table_if_not_exists if EventSourcery::Postgres.config.auto_create_projector_tracker
|
14
14
|
|
15
15
|
unless tracker_table_exists?
|
16
|
-
raise UnableToLockProcessorError,
|
16
|
+
raise UnableToLockProcessorError, 'Projector tracker table does not exist'
|
17
17
|
end
|
18
18
|
|
19
19
|
if processor_name
|
@@ -27,7 +27,7 @@ module EventSourcery
|
|
27
27
|
def processed_event(processor_name, event_id)
|
28
28
|
table.
|
29
29
|
where(name: processor_name.to_s).
|
30
|
-
|
30
|
+
update(last_processed_event_id: event_id)
|
31
31
|
true
|
32
32
|
end
|
33
33
|
|
@@ -44,9 +44,7 @@ module EventSourcery
|
|
44
44
|
|
45
45
|
def last_processed_event_id(processor_name)
|
46
46
|
track_entry = table.where(name: processor_name.to_s).first
|
47
|
-
if track_entry
|
48
|
-
track_entry[:last_processed_event_id]
|
49
|
-
end
|
47
|
+
track_entry[:last_processed_event_id] if track_entry
|
50
48
|
end
|
51
49
|
|
52
50
|
def tracked_processors
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: event_sourcery-postgres
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
|
-
-
|
7
|
+
- Envato
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-06-
|
11
|
+
date: 2017-06-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: sequel
|
@@ -44,14 +44,14 @@ dependencies:
|
|
44
44
|
requirements:
|
45
45
|
- - ">="
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: 0.
|
47
|
+
version: 0.14.0
|
48
48
|
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - ">="
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: 0.
|
54
|
+
version: 0.14.0
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: bundler
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
@@ -124,22 +124,15 @@ dependencies:
|
|
124
124
|
version: '0'
|
125
125
|
description:
|
126
126
|
email:
|
127
|
-
-
|
127
|
+
- rubygems@envato.com
|
128
128
|
executables: []
|
129
129
|
extensions: []
|
130
130
|
extra_rdoc_files: []
|
131
131
|
files:
|
132
|
-
- ".gitignore"
|
133
|
-
- ".rspec"
|
134
|
-
- ".travis.yml"
|
135
132
|
- CHANGELOG.md
|
136
133
|
- CODE_OF_CONDUCT.md
|
137
|
-
- Gemfile
|
138
134
|
- LICENSE.txt
|
139
135
|
- README.md
|
140
|
-
- Rakefile
|
141
|
-
- bin/console
|
142
|
-
- bin/setup
|
143
136
|
- event_sourcery-postgres.gemspec
|
144
137
|
- lib/event_sourcery/postgres.rb
|
145
138
|
- lib/event_sourcery/postgres/config.rb
|
@@ -152,9 +145,6 @@ files:
|
|
152
145
|
- lib/event_sourcery/postgres/table_owner.rb
|
153
146
|
- lib/event_sourcery/postgres/tracker.rb
|
154
147
|
- lib/event_sourcery/postgres/version.rb
|
155
|
-
- script/bench_reading_events.rb
|
156
|
-
- script/bench_writing_events.rb
|
157
|
-
- script/demonstrate_event_sequence_id_gaps.rb
|
158
148
|
homepage: https://github.com/envato/event_sourcery-postgres
|
159
149
|
licenses: []
|
160
150
|
metadata: {}
|
data/.gitignore
DELETED
data/.rspec
DELETED
data/.travis.yml
DELETED
data/Gemfile
DELETED
data/Rakefile
DELETED
data/bin/console
DELETED
@@ -1,14 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
|
3
|
-
require "bundler/setup"
|
4
|
-
require "event_sourcery/postgres"
|
5
|
-
|
6
|
-
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
-
# with your gem easier. You can also use a different console, if you like.
|
8
|
-
|
9
|
-
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
-
# require "pry"
|
11
|
-
# Pry.start
|
12
|
-
|
13
|
-
require "irb"
|
14
|
-
IRB.start(__FILE__)
|
data/bin/setup
DELETED
@@ -1,15 +0,0 @@
|
|
1
|
-
#!/usr/bin/env bash
|
2
|
-
set -euo pipefail
|
3
|
-
IFS=$'\n\t'
|
4
|
-
set -vx
|
5
|
-
|
6
|
-
bundle install
|
7
|
-
|
8
|
-
echo
|
9
|
-
echo "--- Preparing test databases"
|
10
|
-
echo
|
11
|
-
|
12
|
-
dropdb event_sourcery_test || echo 0
|
13
|
-
createdb event_sourcery_test
|
14
|
-
|
15
|
-
# Do any other automated setup that you need to do here
|
@@ -1,63 +0,0 @@
|
|
1
|
-
# Usage:
|
2
|
-
#
|
3
|
-
# ❯ bundle exec ruby script/bench_reading_events.rb
|
4
|
-
# Creating 10000 events
|
5
|
-
# Took 42.35533199999918 to create events
|
6
|
-
# Took 4.9821800000027 to read all events
|
7
|
-
# ^ results from running on a 2016 MacBook
|
8
|
-
|
9
|
-
require 'benchmark'
|
10
|
-
require 'securerandom'
|
11
|
-
require 'sequel'
|
12
|
-
require 'event_sourcery/postgres'
|
13
|
-
|
14
|
-
pg_uri = ENV.fetch('BOXEN_POSTGRESQL_URL') { 'postgres://127.0.0.1:5432/' }.dup
|
15
|
-
pg_uri << 'event_sourcery_test'
|
16
|
-
pg_connection = Sequel.connect(pg_uri)
|
17
|
-
|
18
|
-
EventSourcery.configure do |config|
|
19
|
-
config.postgres.event_store_database = pg_connection
|
20
|
-
config.postgres.projections_database = pg_connection
|
21
|
-
config.logger.level = :fatal
|
22
|
-
end
|
23
|
-
|
24
|
-
def create_events_schema(pg_connection)
|
25
|
-
pg_connection.execute 'drop table if exists events'
|
26
|
-
pg_connection.execute 'drop table if exists aggregates'
|
27
|
-
EventSourcery::Postgres::Schema.create_event_store(db: pg_connection)
|
28
|
-
end
|
29
|
-
|
30
|
-
event_store = EventSourcery::Postgres.config.event_store
|
31
|
-
|
32
|
-
EVENT_TYPES = %i[
|
33
|
-
item_added
|
34
|
-
item_removed
|
35
|
-
item_starred
|
36
|
-
]
|
37
|
-
|
38
|
-
def new_event(uuid)
|
39
|
-
EventSourcery::Event.new(type: EVENT_TYPES.sample,
|
40
|
-
aggregate_id: uuid,
|
41
|
-
body: { 'something' => 'simple' })
|
42
|
-
end
|
43
|
-
|
44
|
-
create_events_schema(pg_connection)
|
45
|
-
|
46
|
-
NUM_EVENTS = 10_000
|
47
|
-
puts "Creating #{NUM_EVENTS} events"
|
48
|
-
time = Benchmark.realtime do
|
49
|
-
uuid = SecureRandom.uuid
|
50
|
-
NUM_EVENTS.times do
|
51
|
-
event_store.sink(new_event(uuid))
|
52
|
-
end
|
53
|
-
end
|
54
|
-
puts "Took #{time} to create events"
|
55
|
-
|
56
|
-
seen_events_count = 0
|
57
|
-
time = Benchmark.realtime do
|
58
|
-
event_store.subscribe(from_id: 0, subscription_master: EventSourcery::EventStore::SignalHandlingSubscriptionMaster.new) do |events|
|
59
|
-
seen_events_count += events.count
|
60
|
-
throw :stop if seen_events_count >= NUM_EVENTS
|
61
|
-
end
|
62
|
-
end
|
63
|
-
puts "Took #{time} to read all events"
|
@@ -1,47 +0,0 @@
|
|
1
|
-
# Usage:
|
2
|
-
#
|
3
|
-
# ❯ bundle exec ruby script/bench_writing_events.rb
|
4
|
-
# Warming up --------------------------------------
|
5
|
-
# event_store.sink
|
6
|
-
# 70.000 i/100ms
|
7
|
-
# Calculating -------------------------------------
|
8
|
-
# event_store.sink
|
9
|
-
# 522.007 (±10.9%) i/s - 2.590k in 5.021909s
|
10
|
-
#
|
11
|
-
# ^ results from running on a 2016 MacBook
|
12
|
-
|
13
|
-
require 'benchmark/ips'
|
14
|
-
require 'securerandom'
|
15
|
-
require 'sequel'
|
16
|
-
require 'event_sourcery/postgres'
|
17
|
-
|
18
|
-
pg_uri = ENV.fetch('BOXEN_POSTGRESQL_URL') { 'postgres://127.0.0.1:5432/' }.dup
|
19
|
-
pg_uri << 'event_sourcery_test'
|
20
|
-
pg_connection = Sequel.connect(pg_uri)
|
21
|
-
|
22
|
-
EventSourcery.configure do |config|
|
23
|
-
config.postgres.event_store_database = pg_connection
|
24
|
-
config.postgres.projections_database = pg_connection
|
25
|
-
config.logger.level = :fatal
|
26
|
-
end
|
27
|
-
|
28
|
-
def create_schema(pg_connection)
|
29
|
-
pg_connection.execute 'drop table if exists events'
|
30
|
-
pg_connection.execute 'drop table if exists aggregates'
|
31
|
-
EventSourcery::Postgres::Schema.create_event_store(db: pg_connection)
|
32
|
-
end
|
33
|
-
|
34
|
-
create_schema(pg_connection)
|
35
|
-
event_store = EventSourcery::Postgres::EventStore.new(pg_connection)
|
36
|
-
|
37
|
-
def new_event
|
38
|
-
EventSourcery::Event.new(type: :item_added,
|
39
|
-
aggregate_id: SecureRandom.uuid,
|
40
|
-
body: { 'something' => 'simple' })
|
41
|
-
end
|
42
|
-
|
43
|
-
Benchmark.ips do |b|
|
44
|
-
b.report("event_store.sink") do
|
45
|
-
event_store.sink(new_event)
|
46
|
-
end
|
47
|
-
end
|
@@ -1,181 +0,0 @@
|
|
1
|
-
# Demonstrates that sequence IDs may not be inserted linearly with concurrent
|
2
|
-
# writers.
|
3
|
-
#
|
4
|
-
# This script writes events in parallel from a number of forked processes,
|
5
|
-
# writing events in a continious loop until the program is interrupted.
|
6
|
-
# The parent process detects gaps in sequence IDs by selecting the last 2
|
7
|
-
# events based on sequence ID. A gap is detected when the 2 IDs returned from
|
8
|
-
# that query aren't sequential. The script will proceed to execute 2 subsequent
|
9
|
-
# queries to see if they show up in the time it takes to complete those before
|
10
|
-
# moving on.
|
11
|
-
#
|
12
|
-
# An easier way to demonstrate this is by using 2 psql consoles:
|
13
|
-
#
|
14
|
-
# - Simulate a transaction taking a long time to commit:
|
15
|
-
# ```
|
16
|
-
# begin;
|
17
|
-
# insert into events (..) values (..);
|
18
|
-
# ```
|
19
|
-
# - Then, in another console:
|
20
|
-
# ```
|
21
|
-
# insert into events (..) values (..);
|
22
|
-
# select * from events;
|
23
|
-
# ```
|
24
|
-
#
|
25
|
-
# The result is that event sequence ID 2 is visible, but only when the first
|
26
|
-
# transaction commits is event sequence ID 1 visible.
|
27
|
-
#
|
28
|
-
# Why does this happen?
|
29
|
-
#
|
30
|
-
# Sequences in Postgres (and most other DBs) are not transactional, changes
|
31
|
-
# to the sequence are visible to other transactions immediately. Also, inserts
|
32
|
-
# from the forked writers may be executed in parallel by postgres.
|
33
|
-
#
|
34
|
-
# The process of inserting into a table that has a sequence or serial column is
|
35
|
-
# to first get the next sequence ID (changing global state), then perform the
|
36
|
-
# insert statement and later commit. In between these 2 steps the sequence ID
|
37
|
-
# is taken but not visible in the table until the insert statement is
|
38
|
-
# committed. Gaps in sequence IDs occur when a process takes a sequence ID and
|
39
|
-
# commits it while another process is in between those 2 steps.
|
40
|
-
#
|
41
|
-
# This means another transaction could have taken the next sequence
|
42
|
-
# ID and committed before that one commits, resulting in a gap in sequence ID's
|
43
|
-
# when reading.
|
44
|
-
#
|
45
|
-
# Why is this a problem?
|
46
|
-
#
|
47
|
-
# Event stream processors use the sequence ID to keep track of where they're up to
|
48
|
-
# in the events table. If a projector processes an event with sequence ID n, it
|
49
|
-
# assumes that the next event it needs to process will have a sequence ID > n.
|
50
|
-
# This approach isn't reliable when sequence IDs appear non-linearly, making it
|
51
|
-
# possible for event stream processors to skip over events.
|
52
|
-
#
|
53
|
-
# How does EventSourcery deal with this?
|
54
|
-
#
|
55
|
-
# EventSourcery uses n transaction level advisory lock to synchronise inserts
|
56
|
-
# to the events table within the writeEvents function. Alternatives:
|
57
|
-
#
|
58
|
-
# - Write events from 1 process only (serialize at the application level)
|
59
|
-
# - Detect gaps when reading events and allow time for in-flight transactions
|
60
|
-
# (the gaps) to commit.
|
61
|
-
# - Built in eventual consistency. Selects would be restricted to events older
|
62
|
-
# than 500ms-1s or the transaction timeout to give enough time for in-flight
|
63
|
-
# transactions to commit.
|
64
|
-
# - Only query events when catching up, after that rely on events to be
|
65
|
-
# delivered through the pub/sub mechanism. Given events would be received out
|
66
|
-
# of order under concurrent writes there's potential for processors to process
|
67
|
-
# a given event twice if they shutdown after processing a sequence that was
|
68
|
-
# part of a gap.
|
69
|
-
#
|
70
|
-
# Usage:
|
71
|
-
#
|
72
|
-
# ❯ bundle exec ruby script/demonstrate_event_sequence_id_gaps.rb
|
73
|
-
# 89847: starting to write events89846: starting to write events
|
74
|
-
|
75
|
-
# 89848: starting to write events
|
76
|
-
# 89849: starting to write events
|
77
|
-
# 89850: starting to write events
|
78
|
-
# GAP: 1 missing sequence IDs. 78 != 76 + 1. Missing events showed up after 1 subsequent query. IDs: [77]
|
79
|
-
# GAP: 1 missing sequence IDs. 168 != 166 + 1. Missing events showed up after 1 subsequent query. IDs: [167]
|
80
|
-
# GAP: 1 missing sequence IDs. 274 != 272 + 1. Missing events showed up after 1 subsequent query. IDs: [273]
|
81
|
-
# GAP: 1 missing sequence IDs. 341 != 339 + 1. Missing events showed up after 1 subsequent query. IDs: [340]
|
82
|
-
# GAP: 1 missing sequence IDs. 461 != 459 + 1. Missing events showed up after 1 subsequent query. IDs: [460]
|
83
|
-
# GAP: 1 missing sequence IDs. 493 != 491 + 1. Missing events showed up after 1 subsequent query. IDs: [492]
|
84
|
-
# GAP: 2 missing sequence IDs. 621 != 618 + 1. Missing events showed up after 1 subsequent query. IDs: [619, 620]
|
85
|
-
|
86
|
-
require 'sequel'
|
87
|
-
require 'securerandom'
|
88
|
-
require 'event_sourcery/postgres'
|
89
|
-
|
90
|
-
def connect
|
91
|
-
pg_uri = ENV.fetch('BOXEN_POSTGRESQL_URL') { 'postgres://127.0.0.1:5432/' }.dup
|
92
|
-
pg_uri << 'event_sourcery_test'
|
93
|
-
Sequel.connect(pg_uri)
|
94
|
-
end
|
95
|
-
|
96
|
-
EventSourcery.logger.level = :info
|
97
|
-
|
98
|
-
def new_event
|
99
|
-
EventSourcery::Event.new(type: :item_added,
|
100
|
-
aggregate_id: SecureRandom.uuid,
|
101
|
-
body: { 'something' => 'simple' })
|
102
|
-
end
|
103
|
-
|
104
|
-
def create_events_schema(db)
|
105
|
-
db.execute 'drop table if exists events'
|
106
|
-
db.execute 'drop table if exists aggregates'
|
107
|
-
EventSourcery::Postgres::Schema.create_event_store(db: db)
|
108
|
-
end
|
109
|
-
|
110
|
-
db = connect
|
111
|
-
create_events_schema(db)
|
112
|
-
db.disconnect
|
113
|
-
sleep 0.3
|
114
|
-
|
115
|
-
NUM_WRITER_PROCESSES = 5
|
116
|
-
NUM_WRITER_PROCESSES.times do
|
117
|
-
fork do |pid|
|
118
|
-
stop = false
|
119
|
-
Signal.trap(:INT) { stop = true }
|
120
|
-
db = connect
|
121
|
-
# when lock_table is set to true an advisory lock is used to synchronise
|
122
|
-
# inserts and no gaps are detected
|
123
|
-
event_store = EventSourcery::Postgres::EventStore.new(db, lock_table: false)
|
124
|
-
puts "#{Process.pid}: starting to write events"
|
125
|
-
until stop
|
126
|
-
event_store.sink(new_event)
|
127
|
-
end
|
128
|
-
end
|
129
|
-
end
|
130
|
-
|
131
|
-
stop = false
|
132
|
-
Signal.trap(:INT) { stop = true }
|
133
|
-
|
134
|
-
def wait_for_missing_ids(db, first_sequence, last_sequence, attempt: 1)
|
135
|
-
missing_ids = db[:events].where(Sequel.lit("id > ? AND id < ?", first_sequence, last_sequence)).order(:id).map {|e| e[:id] }
|
136
|
-
expected_missing_ids = (first_sequence+1)..(last_sequence-1)
|
137
|
-
if missing_ids == expected_missing_ids.to_a
|
138
|
-
print "Missing events showed up after #{attempt} subsequent query. IDs: #{missing_ids}"
|
139
|
-
else
|
140
|
-
if attempt < 2
|
141
|
-
wait_for_missing_ids(db, first_sequence, last_sequence, attempt: attempt + 1)
|
142
|
-
else
|
143
|
-
print "Missing events didn't show up after #{attempt} subsequent queries"
|
144
|
-
end
|
145
|
-
end
|
146
|
-
end
|
147
|
-
|
148
|
-
until stop
|
149
|
-
|
150
|
-
# query for the last 2 sequences in the events table
|
151
|
-
first_sequence, last_sequence = *db[:events].
|
152
|
-
order(Sequel.desc(:id)).
|
153
|
-
select(:id).
|
154
|
-
limit(2).
|
155
|
-
map { |e| e[:id] }.
|
156
|
-
reverse
|
157
|
-
|
158
|
-
next if first_sequence.nil? || last_sequence.nil?
|
159
|
-
|
160
|
-
if last_sequence != first_sequence + 1
|
161
|
-
num_missing = last_sequence - first_sequence - 1
|
162
|
-
print "GAP: #{num_missing} missing sequence IDs. #{last_sequence} != #{first_sequence} + 1. "
|
163
|
-
wait_for_missing_ids(db, first_sequence, last_sequence)
|
164
|
-
puts
|
165
|
-
end
|
166
|
-
end
|
167
|
-
|
168
|
-
Process.waitall
|
169
|
-
|
170
|
-
puts
|
171
|
-
puts "Looking for gaps in sequence IDs in events table:"
|
172
|
-
ids = db[:events].select(:id).order(:id).all.map { |e| e[:id] }
|
173
|
-
expected_ids = (ids.min..ids.max).to_a
|
174
|
-
missing_ids = (expected_ids - ids)
|
175
|
-
if missing_ids.empty?
|
176
|
-
puts "No remaining gaps"
|
177
|
-
else
|
178
|
-
missing_ids.each do |id|
|
179
|
-
puts "Unable to find row with sequence ID #{id}"
|
180
|
-
end
|
181
|
-
end
|