event_sourcery-postgres 0.3.0 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +13 -0
- data/event_sourcery-postgres.gemspec +15 -15
- data/lib/event_sourcery/postgres/event_store.rb +11 -19
- data/lib/event_sourcery/postgres/optimised_event_poll_waiter.rb +13 -17
- data/lib/event_sourcery/postgres/projector.rb +4 -4
- data/lib/event_sourcery/postgres/queue_with_interval_callback.rb +2 -2
- data/lib/event_sourcery/postgres/reactor.rb +5 -7
- data/lib/event_sourcery/postgres/schema.rb +3 -1
- data/lib/event_sourcery/postgres/table_owner.rb +1 -1
- data/lib/event_sourcery/postgres/tracker.rb +4 -6
- data/lib/event_sourcery/postgres/version.rb +1 -1
- metadata +6 -16
- data/.gitignore +0 -13
- data/.rspec +0 -3
- data/.travis.yml +0 -12
- data/Gemfile +0 -5
- data/Rakefile +0 -6
- data/bin/console +0 -14
- data/bin/setup +0 -15
- data/script/bench_reading_events.rb +0 -63
- data/script/bench_writing_events.rb +0 -47
- data/script/demonstrate_event_sequence_id_gaps.rb +0 -181
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e45ae5ad105bd165a2fb4b59be6032488ec925df
|
4
|
+
data.tar.gz: c36491b6e980803520d2b3a900207500f694b500
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a86c88d7db9e0368d3ddbea1b6b7ce869d8f3465dd1bce3990f0038edffcb5712da10d4182fd0ce46427b2e3aa08b67b7da6712d7aec0abd16f766741a492556
|
7
|
+
data.tar.gz: 3f286cd47b3eb36e3b3e65929d6df0d51a1bc8762f1256d1b21c6f8041861fc8087bde5978989c7abb58d6f274add9dca368ca992105a12e5e7af6069e390dc7
|
data/CHANGELOG.md
CHANGED
@@ -6,6 +6,19 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
|
|
6
6
|
|
7
7
|
## [Unreleased]
|
8
8
|
|
9
|
+
## [0.4.0] - 2017-6-21
|
10
|
+
### Changed
|
11
|
+
- Reactors store the UUID of the event being processed in the `causation_id`
|
12
|
+
of any emitted events. This replaces the old behaviour of storing id of the
|
13
|
+
event being processed in a `_driven_by_event_id` attribute in the emitted
|
14
|
+
event's body.
|
15
|
+
|
16
|
+
### Added
|
17
|
+
- Reactors store the correlation id of the event being processed in the
|
18
|
+
`correlation_id` of any emitted events.
|
19
|
+
- Added index on the `events` table for `correlation_id` and `causation_id`
|
20
|
+
columns.
|
21
|
+
|
9
22
|
## [0.3.0] - 2017-6-16
|
10
23
|
### Changed
|
11
24
|
- The event store persists the event `correlation_id` and `causation_id`.
|
@@ -4,30 +4,30 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
require 'event_sourcery/postgres/version'
|
5
5
|
|
6
6
|
Gem::Specification.new do |spec|
|
7
|
-
spec.name =
|
7
|
+
spec.name = 'event_sourcery-postgres'
|
8
8
|
spec.version = EventSourcery::Postgres::VERSION
|
9
|
-
# TODO: update authors
|
10
|
-
spec.authors = ["Steve Hodgkiss"]
|
11
|
-
spec.email = ["steve@hodgkiss.me"]
|
12
9
|
|
13
|
-
spec.
|
14
|
-
spec.
|
10
|
+
spec.authors = ['Envato']
|
11
|
+
spec.email = ['rubygems@envato.com']
|
12
|
+
|
13
|
+
spec.summary = 'Postgres event store for use with EventSourcery'
|
14
|
+
spec.homepage = 'https://github.com/envato/event_sourcery-postgres'
|
15
15
|
|
16
16
|
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
17
|
-
f.match(%r{^(
|
17
|
+
f.match(%r{^(\.|bin/|Gemfile|Rakefile|script/|spec/)})
|
18
18
|
end
|
19
|
-
spec.bindir =
|
19
|
+
spec.bindir = 'exe'
|
20
20
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
21
|
-
spec.require_paths = [
|
21
|
+
spec.require_paths = ['lib']
|
22
22
|
|
23
23
|
spec.required_ruby_version = '>= 2.2.0'
|
24
24
|
|
25
25
|
spec.add_dependency 'sequel', '~> 4.38'
|
26
26
|
spec.add_dependency 'pg'
|
27
|
-
spec.add_dependency 'event_sourcery', '>= 0.
|
28
|
-
spec.add_development_dependency
|
29
|
-
spec.add_development_dependency
|
30
|
-
spec.add_development_dependency
|
31
|
-
spec.add_development_dependency
|
32
|
-
spec.add_development_dependency
|
27
|
+
spec.add_dependency 'event_sourcery', '>= 0.14.0'
|
28
|
+
spec.add_development_dependency 'bundler', '~> 1.10'
|
29
|
+
spec.add_development_dependency 'rake', '~> 10.0'
|
30
|
+
spec.add_development_dependency 'rspec', '~> 3.0'
|
31
|
+
spec.add_development_dependency 'pry'
|
32
|
+
spec.add_development_dependency 'benchmark-ips'
|
33
33
|
end
|
@@ -36,19 +36,13 @@ module EventSourcery
|
|
36
36
|
order(:id).
|
37
37
|
where(Sequel.lit('id >= ?', id)).
|
38
38
|
limit(limit)
|
39
|
-
if event_types
|
40
|
-
|
41
|
-
end
|
42
|
-
query.map do |event_row|
|
43
|
-
build_event(event_row)
|
44
|
-
end
|
39
|
+
query = query.where(type: event_types) if event_types
|
40
|
+
query.map { |event_row| build_event(event_row) }
|
45
41
|
end
|
46
42
|
|
47
43
|
def latest_event_id(event_types: nil)
|
48
44
|
latest_event = events_table
|
49
|
-
if event_types
|
50
|
-
latest_event = latest_event.where(type: event_types)
|
51
|
-
end
|
45
|
+
latest_event = latest_event.where(type: event_types) if event_types
|
52
46
|
latest_event = latest_event.order(:id).last
|
53
47
|
if latest_event
|
54
48
|
latest_event[:id]
|
@@ -74,9 +68,7 @@ module EventSourcery
|
|
74
68
|
subscription_master: subscription_master,
|
75
69
|
on_new_events: block
|
76
70
|
}
|
77
|
-
EventSourcery::EventStore::Subscription.new(args).tap
|
78
|
-
s.start
|
79
|
-
end
|
71
|
+
EventSourcery::EventStore::Subscription.new(args).tap(&:start)
|
80
72
|
end
|
81
73
|
|
82
74
|
private
|
@@ -113,7 +105,7 @@ module EventSourcery
|
|
113
105
|
|
114
106
|
def sql_literal_array(events, type, &block)
|
115
107
|
sql_array = events.map do |event|
|
116
|
-
|
108
|
+
to_sql_literal(block.call(event))
|
117
109
|
end.join(', ')
|
118
110
|
"array[#{sql_array}]::#{type}[]"
|
119
111
|
end
|
@@ -125,12 +117,12 @@ module EventSourcery
|
|
125
117
|
def to_sql_literal(value)
|
126
118
|
return 'null' unless value
|
127
119
|
wrapped_value = if Time === value
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
120
|
+
value.iso8601(6)
|
121
|
+
elsif Hash === value
|
122
|
+
Sequel.pg_json(value)
|
123
|
+
else
|
124
|
+
value
|
125
|
+
end
|
134
126
|
@pg_connection.literal(wrapped_value)
|
135
127
|
end
|
136
128
|
|
@@ -4,7 +4,7 @@ module EventSourcery
|
|
4
4
|
class OptimisedEventPollWaiter
|
5
5
|
ListenThreadDied = Class.new(StandardError)
|
6
6
|
|
7
|
-
def initialize(pg_connection:, timeout: 30, after_listen: proc {
|
7
|
+
def initialize(pg_connection:, timeout: 30, after_listen: proc {})
|
8
8
|
@pg_connection = pg_connection
|
9
9
|
@timeout = timeout
|
10
10
|
@events_queue = QueueWithIntervalCallback.new
|
@@ -17,7 +17,7 @@ module EventSourcery
|
|
17
17
|
block.call
|
18
18
|
end
|
19
19
|
start_async(after_listen: after_listen)
|
20
|
-
catch(:stop)
|
20
|
+
catch(:stop) do
|
21
21
|
block.call
|
22
22
|
loop do
|
23
23
|
ensure_listen_thread_alive!
|
@@ -25,7 +25,7 @@ module EventSourcery
|
|
25
25
|
clear_new_event_queue
|
26
26
|
block.call
|
27
27
|
end
|
28
|
-
|
28
|
+
end
|
29
29
|
ensure
|
30
30
|
shutdown!
|
31
31
|
end
|
@@ -33,15 +33,11 @@ module EventSourcery
|
|
33
33
|
private
|
34
34
|
|
35
35
|
def shutdown!
|
36
|
-
if @listen_thread.alive?
|
37
|
-
@listen_thread.kill
|
38
|
-
end
|
36
|
+
@listen_thread.kill if @listen_thread.alive?
|
39
37
|
end
|
40
38
|
|
41
39
|
def ensure_listen_thread_alive!
|
42
|
-
|
43
|
-
raise ListenThreadDied
|
44
|
-
end
|
40
|
+
raise ListenThreadDied unless @listen_thread.alive?
|
45
41
|
end
|
46
42
|
|
47
43
|
def wait_for_new_event_to_appear
|
@@ -54,16 +50,18 @@ module EventSourcery
|
|
54
50
|
|
55
51
|
def start_async(after_listen: nil)
|
56
52
|
after_listen_callback = if after_listen
|
57
|
-
proc
|
53
|
+
proc do
|
58
54
|
after_listen.call
|
59
55
|
@after_listen.call if @after_listen
|
60
|
-
|
56
|
+
end
|
61
57
|
else
|
62
58
|
@after_listen
|
63
59
|
end
|
64
|
-
@listen_thread = Thread.new
|
65
|
-
|
66
|
-
|
60
|
+
@listen_thread = Thread.new do
|
61
|
+
listen_for_new_events(loop: true,
|
62
|
+
after_listen: after_listen_callback,
|
63
|
+
timeout: @timeout)
|
64
|
+
end
|
67
65
|
end
|
68
66
|
|
69
67
|
def listen_for_new_events(loop: true, after_listen: nil, timeout: 30)
|
@@ -71,9 +69,7 @@ module EventSourcery
|
|
71
69
|
loop: loop,
|
72
70
|
after_listen: after_listen,
|
73
71
|
timeout: timeout) do |_channel, _pid, _payload|
|
74
|
-
if @events_queue.empty?
|
75
|
-
@events_queue.push(:new_event_arrived)
|
76
|
-
end
|
72
|
+
@events_queue.push(:new_event_arrived) if @events_queue.empty?
|
77
73
|
end
|
78
74
|
end
|
79
75
|
end
|
@@ -6,12 +6,12 @@ module EventSourcery
|
|
6
6
|
base.prepend(TableOwner)
|
7
7
|
base.include(InstanceMethods)
|
8
8
|
base.class_eval do
|
9
|
-
|
9
|
+
alias_method :project, :process
|
10
10
|
|
11
11
|
class << self
|
12
|
-
|
13
|
-
|
14
|
-
|
12
|
+
alias_method :project, :process
|
13
|
+
alias_method :projects_events, :processes_events
|
14
|
+
alias_method :projector_name, :processor_name
|
15
15
|
end
|
16
16
|
end
|
17
17
|
end
|
@@ -3,7 +3,7 @@ module EventSourcery
|
|
3
3
|
class QueueWithIntervalCallback < ::Queue
|
4
4
|
attr_accessor :callback
|
5
5
|
|
6
|
-
def initialize(callback: proc {
|
6
|
+
def initialize(callback: proc {}, callback_interval: EventSourcery::Postgres.config.callback_interval_if_no_new_events, poll_interval: 0.1)
|
7
7
|
@callback = callback
|
8
8
|
@callback_interval = callback_interval
|
9
9
|
@poll_interval = poll_interval
|
@@ -20,7 +20,7 @@ module EventSourcery
|
|
20
20
|
def pop_with_interval_callback
|
21
21
|
time = Time.now
|
22
22
|
loop do
|
23
|
-
return pop(true)
|
23
|
+
return pop(true) unless empty?
|
24
24
|
if @callback_interval && Time.now > time + @callback_interval
|
25
25
|
@callback.call
|
26
26
|
time = Time.now
|
@@ -45,20 +45,18 @@ module EventSourcery
|
|
45
45
|
end
|
46
46
|
end
|
47
47
|
|
48
|
-
DRIVEN_BY_EVENT_PAYLOAD_KEY = :_driven_by_event_id
|
49
|
-
|
50
48
|
private
|
51
49
|
|
52
50
|
attr_reader :event_sink, :event_source
|
53
51
|
|
54
52
|
def emit_event(event_or_hash, &block)
|
55
53
|
event = if Event === event_or_hash
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
54
|
+
event_or_hash
|
55
|
+
else
|
56
|
+
Event.new(event_or_hash)
|
57
|
+
end
|
60
58
|
raise UndeclaredEventEmissionError unless self.class.emits_event?(event.class)
|
61
|
-
event.
|
59
|
+
event = event.with(causation_id: _event.uuid, correlation_id: _event.correlation_id)
|
62
60
|
invoke_action_and_emit_event(event, block)
|
63
61
|
EventSourcery.logger.debug { "[#{self.processor_name}] Emitted event: #{event.inspect}" }
|
64
62
|
end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
module EventSourcery
|
2
2
|
module Postgres
|
3
3
|
module Schema
|
4
|
-
|
4
|
+
module_function
|
5
5
|
|
6
6
|
def create_event_store(db: EventSourcery::Postgres.config.event_store_database,
|
7
7
|
events_table_name: EventSourcery::Postgres.config.events_table_name,
|
@@ -28,6 +28,8 @@ module EventSourcery
|
|
28
28
|
index [:aggregate_id, :version], unique: true
|
29
29
|
index :uuid, unique: true
|
30
30
|
index :type
|
31
|
+
index :correlation_id
|
32
|
+
index :causation_id
|
31
33
|
index :created_at
|
32
34
|
end
|
33
35
|
end
|
@@ -5,7 +5,7 @@ module EventSourcery
|
|
5
5
|
table_name: EventSourcery::Postgres.config.tracker_table_name,
|
6
6
|
obtain_processor_lock: true)
|
7
7
|
@connection = connection
|
8
|
-
@table_name = table_name
|
8
|
+
@table_name = table_name.to_sym
|
9
9
|
@obtain_processor_lock = obtain_processor_lock
|
10
10
|
end
|
11
11
|
|
@@ -13,7 +13,7 @@ module EventSourcery
|
|
13
13
|
create_table_if_not_exists if EventSourcery::Postgres.config.auto_create_projector_tracker
|
14
14
|
|
15
15
|
unless tracker_table_exists?
|
16
|
-
raise UnableToLockProcessorError,
|
16
|
+
raise UnableToLockProcessorError, 'Projector tracker table does not exist'
|
17
17
|
end
|
18
18
|
|
19
19
|
if processor_name
|
@@ -27,7 +27,7 @@ module EventSourcery
|
|
27
27
|
def processed_event(processor_name, event_id)
|
28
28
|
table.
|
29
29
|
where(name: processor_name.to_s).
|
30
|
-
|
30
|
+
update(last_processed_event_id: event_id)
|
31
31
|
true
|
32
32
|
end
|
33
33
|
|
@@ -44,9 +44,7 @@ module EventSourcery
|
|
44
44
|
|
45
45
|
def last_processed_event_id(processor_name)
|
46
46
|
track_entry = table.where(name: processor_name.to_s).first
|
47
|
-
if track_entry
|
48
|
-
track_entry[:last_processed_event_id]
|
49
|
-
end
|
47
|
+
track_entry[:last_processed_event_id] if track_entry
|
50
48
|
end
|
51
49
|
|
52
50
|
def tracked_processors
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: event_sourcery-postgres
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
|
-
-
|
7
|
+
- Envato
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-06-
|
11
|
+
date: 2017-06-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: sequel
|
@@ -44,14 +44,14 @@ dependencies:
|
|
44
44
|
requirements:
|
45
45
|
- - ">="
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: 0.
|
47
|
+
version: 0.14.0
|
48
48
|
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - ">="
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: 0.
|
54
|
+
version: 0.14.0
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: bundler
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
@@ -124,22 +124,15 @@ dependencies:
|
|
124
124
|
version: '0'
|
125
125
|
description:
|
126
126
|
email:
|
127
|
-
-
|
127
|
+
- rubygems@envato.com
|
128
128
|
executables: []
|
129
129
|
extensions: []
|
130
130
|
extra_rdoc_files: []
|
131
131
|
files:
|
132
|
-
- ".gitignore"
|
133
|
-
- ".rspec"
|
134
|
-
- ".travis.yml"
|
135
132
|
- CHANGELOG.md
|
136
133
|
- CODE_OF_CONDUCT.md
|
137
|
-
- Gemfile
|
138
134
|
- LICENSE.txt
|
139
135
|
- README.md
|
140
|
-
- Rakefile
|
141
|
-
- bin/console
|
142
|
-
- bin/setup
|
143
136
|
- event_sourcery-postgres.gemspec
|
144
137
|
- lib/event_sourcery/postgres.rb
|
145
138
|
- lib/event_sourcery/postgres/config.rb
|
@@ -152,9 +145,6 @@ files:
|
|
152
145
|
- lib/event_sourcery/postgres/table_owner.rb
|
153
146
|
- lib/event_sourcery/postgres/tracker.rb
|
154
147
|
- lib/event_sourcery/postgres/version.rb
|
155
|
-
- script/bench_reading_events.rb
|
156
|
-
- script/bench_writing_events.rb
|
157
|
-
- script/demonstrate_event_sequence_id_gaps.rb
|
158
148
|
homepage: https://github.com/envato/event_sourcery-postgres
|
159
149
|
licenses: []
|
160
150
|
metadata: {}
|
data/.gitignore
DELETED
data/.rspec
DELETED
data/.travis.yml
DELETED
data/Gemfile
DELETED
data/Rakefile
DELETED
data/bin/console
DELETED
@@ -1,14 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
|
3
|
-
require "bundler/setup"
|
4
|
-
require "event_sourcery/postgres"
|
5
|
-
|
6
|
-
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
-
# with your gem easier. You can also use a different console, if you like.
|
8
|
-
|
9
|
-
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
-
# require "pry"
|
11
|
-
# Pry.start
|
12
|
-
|
13
|
-
require "irb"
|
14
|
-
IRB.start(__FILE__)
|
data/bin/setup
DELETED
@@ -1,15 +0,0 @@
|
|
1
|
-
#!/usr/bin/env bash
|
2
|
-
set -euo pipefail
|
3
|
-
IFS=$'\n\t'
|
4
|
-
set -vx
|
5
|
-
|
6
|
-
bundle install
|
7
|
-
|
8
|
-
echo
|
9
|
-
echo "--- Preparing test databases"
|
10
|
-
echo
|
11
|
-
|
12
|
-
dropdb event_sourcery_test || echo 0
|
13
|
-
createdb event_sourcery_test
|
14
|
-
|
15
|
-
# Do any other automated setup that you need to do here
|
@@ -1,63 +0,0 @@
|
|
1
|
-
# Usage:
|
2
|
-
#
|
3
|
-
# ❯ bundle exec ruby script/bench_reading_events.rb
|
4
|
-
# Creating 10000 events
|
5
|
-
# Took 42.35533199999918 to create events
|
6
|
-
# Took 4.9821800000027 to read all events
|
7
|
-
# ^ results from running on a 2016 MacBook
|
8
|
-
|
9
|
-
require 'benchmark'
|
10
|
-
require 'securerandom'
|
11
|
-
require 'sequel'
|
12
|
-
require 'event_sourcery/postgres'
|
13
|
-
|
14
|
-
pg_uri = ENV.fetch('BOXEN_POSTGRESQL_URL') { 'postgres://127.0.0.1:5432/' }.dup
|
15
|
-
pg_uri << 'event_sourcery_test'
|
16
|
-
pg_connection = Sequel.connect(pg_uri)
|
17
|
-
|
18
|
-
EventSourcery.configure do |config|
|
19
|
-
config.postgres.event_store_database = pg_connection
|
20
|
-
config.postgres.projections_database = pg_connection
|
21
|
-
config.logger.level = :fatal
|
22
|
-
end
|
23
|
-
|
24
|
-
def create_events_schema(pg_connection)
|
25
|
-
pg_connection.execute 'drop table if exists events'
|
26
|
-
pg_connection.execute 'drop table if exists aggregates'
|
27
|
-
EventSourcery::Postgres::Schema.create_event_store(db: pg_connection)
|
28
|
-
end
|
29
|
-
|
30
|
-
event_store = EventSourcery::Postgres.config.event_store
|
31
|
-
|
32
|
-
EVENT_TYPES = %i[
|
33
|
-
item_added
|
34
|
-
item_removed
|
35
|
-
item_starred
|
36
|
-
]
|
37
|
-
|
38
|
-
def new_event(uuid)
|
39
|
-
EventSourcery::Event.new(type: EVENT_TYPES.sample,
|
40
|
-
aggregate_id: uuid,
|
41
|
-
body: { 'something' => 'simple' })
|
42
|
-
end
|
43
|
-
|
44
|
-
create_events_schema(pg_connection)
|
45
|
-
|
46
|
-
NUM_EVENTS = 10_000
|
47
|
-
puts "Creating #{NUM_EVENTS} events"
|
48
|
-
time = Benchmark.realtime do
|
49
|
-
uuid = SecureRandom.uuid
|
50
|
-
NUM_EVENTS.times do
|
51
|
-
event_store.sink(new_event(uuid))
|
52
|
-
end
|
53
|
-
end
|
54
|
-
puts "Took #{time} to create events"
|
55
|
-
|
56
|
-
seen_events_count = 0
|
57
|
-
time = Benchmark.realtime do
|
58
|
-
event_store.subscribe(from_id: 0, subscription_master: EventSourcery::EventStore::SignalHandlingSubscriptionMaster.new) do |events|
|
59
|
-
seen_events_count += events.count
|
60
|
-
throw :stop if seen_events_count >= NUM_EVENTS
|
61
|
-
end
|
62
|
-
end
|
63
|
-
puts "Took #{time} to read all events"
|
@@ -1,47 +0,0 @@
|
|
1
|
-
# Usage:
|
2
|
-
#
|
3
|
-
# ❯ bundle exec ruby script/bench_writing_events.rb
|
4
|
-
# Warming up --------------------------------------
|
5
|
-
# event_store.sink
|
6
|
-
# 70.000 i/100ms
|
7
|
-
# Calculating -------------------------------------
|
8
|
-
# event_store.sink
|
9
|
-
# 522.007 (±10.9%) i/s - 2.590k in 5.021909s
|
10
|
-
#
|
11
|
-
# ^ results from running on a 2016 MacBook
|
12
|
-
|
13
|
-
require 'benchmark/ips'
|
14
|
-
require 'securerandom'
|
15
|
-
require 'sequel'
|
16
|
-
require 'event_sourcery/postgres'
|
17
|
-
|
18
|
-
pg_uri = ENV.fetch('BOXEN_POSTGRESQL_URL') { 'postgres://127.0.0.1:5432/' }.dup
|
19
|
-
pg_uri << 'event_sourcery_test'
|
20
|
-
pg_connection = Sequel.connect(pg_uri)
|
21
|
-
|
22
|
-
EventSourcery.configure do |config|
|
23
|
-
config.postgres.event_store_database = pg_connection
|
24
|
-
config.postgres.projections_database = pg_connection
|
25
|
-
config.logger.level = :fatal
|
26
|
-
end
|
27
|
-
|
28
|
-
def create_schema(pg_connection)
|
29
|
-
pg_connection.execute 'drop table if exists events'
|
30
|
-
pg_connection.execute 'drop table if exists aggregates'
|
31
|
-
EventSourcery::Postgres::Schema.create_event_store(db: pg_connection)
|
32
|
-
end
|
33
|
-
|
34
|
-
create_schema(pg_connection)
|
35
|
-
event_store = EventSourcery::Postgres::EventStore.new(pg_connection)
|
36
|
-
|
37
|
-
def new_event
|
38
|
-
EventSourcery::Event.new(type: :item_added,
|
39
|
-
aggregate_id: SecureRandom.uuid,
|
40
|
-
body: { 'something' => 'simple' })
|
41
|
-
end
|
42
|
-
|
43
|
-
Benchmark.ips do |b|
|
44
|
-
b.report("event_store.sink") do
|
45
|
-
event_store.sink(new_event)
|
46
|
-
end
|
47
|
-
end
|
@@ -1,181 +0,0 @@
|
|
1
|
-
# Demonstrates that sequence IDs may not be inserted linearly with concurrent
|
2
|
-
# writers.
|
3
|
-
#
|
4
|
-
# This script writes events in parallel from a number of forked processes,
|
5
|
-
# writing events in a continious loop until the program is interrupted.
|
6
|
-
# The parent process detects gaps in sequence IDs by selecting the last 2
|
7
|
-
# events based on sequence ID. A gap is detected when the 2 IDs returned from
|
8
|
-
# that query aren't sequential. The script will proceed to execute 2 subsequent
|
9
|
-
# queries to see if they show up in the time it takes to complete those before
|
10
|
-
# moving on.
|
11
|
-
#
|
12
|
-
# An easier way to demonstrate this is by using 2 psql consoles:
|
13
|
-
#
|
14
|
-
# - Simulate a transaction taking a long time to commit:
|
15
|
-
# ```
|
16
|
-
# begin;
|
17
|
-
# insert into events (..) values (..);
|
18
|
-
# ```
|
19
|
-
# - Then, in another console:
|
20
|
-
# ```
|
21
|
-
# insert into events (..) values (..);
|
22
|
-
# select * from events;
|
23
|
-
# ```
|
24
|
-
#
|
25
|
-
# The result is that event sequence ID 2 is visible, but only when the first
|
26
|
-
# transaction commits is event sequence ID 1 visible.
|
27
|
-
#
|
28
|
-
# Why does this happen?
|
29
|
-
#
|
30
|
-
# Sequences in Postgres (and most other DBs) are not transactional, changes
|
31
|
-
# to the sequence are visible to other transactions immediately. Also, inserts
|
32
|
-
# from the forked writers may be executed in parallel by postgres.
|
33
|
-
#
|
34
|
-
# The process of inserting into a table that has a sequence or serial column is
|
35
|
-
# to first get the next sequence ID (changing global state), then perform the
|
36
|
-
# insert statement and later commit. In between these 2 steps the sequence ID
|
37
|
-
# is taken but not visible in the table until the insert statement is
|
38
|
-
# committed. Gaps in sequence IDs occur when a process takes a sequence ID and
|
39
|
-
# commits it while another process is in between those 2 steps.
|
40
|
-
#
|
41
|
-
# This means another transaction could have taken the next sequence
|
42
|
-
# ID and committed before that one commits, resulting in a gap in sequence ID's
|
43
|
-
# when reading.
|
44
|
-
#
|
45
|
-
# Why is this a problem?
|
46
|
-
#
|
47
|
-
# Event stream processors use the sequence ID to keep track of where they're up to
|
48
|
-
# in the events table. If a projector processes an event with sequence ID n, it
|
49
|
-
# assumes that the next event it needs to process will have a sequence ID > n.
|
50
|
-
# This approach isn't reliable when sequence IDs appear non-linearly, making it
|
51
|
-
# possible for event stream processors to skip over events.
|
52
|
-
#
|
53
|
-
# How does EventSourcery deal with this?
|
54
|
-
#
|
55
|
-
# EventSourcery uses n transaction level advisory lock to synchronise inserts
|
56
|
-
# to the events table within the writeEvents function. Alternatives:
|
57
|
-
#
|
58
|
-
# - Write events from 1 process only (serialize at the application level)
|
59
|
-
# - Detect gaps when reading events and allow time for in-flight transactions
|
60
|
-
# (the gaps) to commit.
|
61
|
-
# - Built in eventual consistency. Selects would be restricted to events older
|
62
|
-
# than 500ms-1s or the transaction timeout to give enough time for in-flight
|
63
|
-
# transactions to commit.
|
64
|
-
# - Only query events when catching up, after that rely on events to be
|
65
|
-
# delivered through the pub/sub mechanism. Given events would be received out
|
66
|
-
# of order under concurrent writes there's potential for processors to process
|
67
|
-
# a given event twice if they shutdown after processing a sequence that was
|
68
|
-
# part of a gap.
|
69
|
-
#
|
70
|
-
# Usage:
|
71
|
-
#
|
72
|
-
# ❯ bundle exec ruby script/demonstrate_event_sequence_id_gaps.rb
|
73
|
-
# 89847: starting to write events89846: starting to write events
|
74
|
-
|
75
|
-
# 89848: starting to write events
|
76
|
-
# 89849: starting to write events
|
77
|
-
# 89850: starting to write events
|
78
|
-
# GAP: 1 missing sequence IDs. 78 != 76 + 1. Missing events showed up after 1 subsequent query. IDs: [77]
|
79
|
-
# GAP: 1 missing sequence IDs. 168 != 166 + 1. Missing events showed up after 1 subsequent query. IDs: [167]
|
80
|
-
# GAP: 1 missing sequence IDs. 274 != 272 + 1. Missing events showed up after 1 subsequent query. IDs: [273]
|
81
|
-
# GAP: 1 missing sequence IDs. 341 != 339 + 1. Missing events showed up after 1 subsequent query. IDs: [340]
|
82
|
-
# GAP: 1 missing sequence IDs. 461 != 459 + 1. Missing events showed up after 1 subsequent query. IDs: [460]
|
83
|
-
# GAP: 1 missing sequence IDs. 493 != 491 + 1. Missing events showed up after 1 subsequent query. IDs: [492]
|
84
|
-
# GAP: 2 missing sequence IDs. 621 != 618 + 1. Missing events showed up after 1 subsequent query. IDs: [619, 620]
|
85
|
-
|
86
|
-
require 'sequel'
|
87
|
-
require 'securerandom'
|
88
|
-
require 'event_sourcery/postgres'
|
89
|
-
|
90
|
-
def connect
|
91
|
-
pg_uri = ENV.fetch('BOXEN_POSTGRESQL_URL') { 'postgres://127.0.0.1:5432/' }.dup
|
92
|
-
pg_uri << 'event_sourcery_test'
|
93
|
-
Sequel.connect(pg_uri)
|
94
|
-
end
|
95
|
-
|
96
|
-
EventSourcery.logger.level = :info
|
97
|
-
|
98
|
-
def new_event
|
99
|
-
EventSourcery::Event.new(type: :item_added,
|
100
|
-
aggregate_id: SecureRandom.uuid,
|
101
|
-
body: { 'something' => 'simple' })
|
102
|
-
end
|
103
|
-
|
104
|
-
def create_events_schema(db)
|
105
|
-
db.execute 'drop table if exists events'
|
106
|
-
db.execute 'drop table if exists aggregates'
|
107
|
-
EventSourcery::Postgres::Schema.create_event_store(db: db)
|
108
|
-
end
|
109
|
-
|
110
|
-
db = connect
|
111
|
-
create_events_schema(db)
|
112
|
-
db.disconnect
|
113
|
-
sleep 0.3
|
114
|
-
|
115
|
-
NUM_WRITER_PROCESSES = 5
|
116
|
-
NUM_WRITER_PROCESSES.times do
|
117
|
-
fork do |pid|
|
118
|
-
stop = false
|
119
|
-
Signal.trap(:INT) { stop = true }
|
120
|
-
db = connect
|
121
|
-
# when lock_table is set to true an advisory lock is used to synchronise
|
122
|
-
# inserts and no gaps are detected
|
123
|
-
event_store = EventSourcery::Postgres::EventStore.new(db, lock_table: false)
|
124
|
-
puts "#{Process.pid}: starting to write events"
|
125
|
-
until stop
|
126
|
-
event_store.sink(new_event)
|
127
|
-
end
|
128
|
-
end
|
129
|
-
end
|
130
|
-
|
131
|
-
stop = false
|
132
|
-
Signal.trap(:INT) { stop = true }
|
133
|
-
|
134
|
-
def wait_for_missing_ids(db, first_sequence, last_sequence, attempt: 1)
|
135
|
-
missing_ids = db[:events].where(Sequel.lit("id > ? AND id < ?", first_sequence, last_sequence)).order(:id).map {|e| e[:id] }
|
136
|
-
expected_missing_ids = (first_sequence+1)..(last_sequence-1)
|
137
|
-
if missing_ids == expected_missing_ids.to_a
|
138
|
-
print "Missing events showed up after #{attempt} subsequent query. IDs: #{missing_ids}"
|
139
|
-
else
|
140
|
-
if attempt < 2
|
141
|
-
wait_for_missing_ids(db, first_sequence, last_sequence, attempt: attempt + 1)
|
142
|
-
else
|
143
|
-
print "Missing events didn't show up after #{attempt} subsequent queries"
|
144
|
-
end
|
145
|
-
end
|
146
|
-
end
|
147
|
-
|
148
|
-
until stop
|
149
|
-
|
150
|
-
# query for the last 2 sequences in the events table
|
151
|
-
first_sequence, last_sequence = *db[:events].
|
152
|
-
order(Sequel.desc(:id)).
|
153
|
-
select(:id).
|
154
|
-
limit(2).
|
155
|
-
map { |e| e[:id] }.
|
156
|
-
reverse
|
157
|
-
|
158
|
-
next if first_sequence.nil? || last_sequence.nil?
|
159
|
-
|
160
|
-
if last_sequence != first_sequence + 1
|
161
|
-
num_missing = last_sequence - first_sequence - 1
|
162
|
-
print "GAP: #{num_missing} missing sequence IDs. #{last_sequence} != #{first_sequence} + 1. "
|
163
|
-
wait_for_missing_ids(db, first_sequence, last_sequence)
|
164
|
-
puts
|
165
|
-
end
|
166
|
-
end
|
167
|
-
|
168
|
-
Process.waitall
|
169
|
-
|
170
|
-
puts
|
171
|
-
puts "Looking for gaps in sequence IDs in events table:"
|
172
|
-
ids = db[:events].select(:id).order(:id).all.map { |e| e[:id] }
|
173
|
-
expected_ids = (ids.min..ids.max).to_a
|
174
|
-
missing_ids = (expected_ids - ids)
|
175
|
-
if missing_ids.empty?
|
176
|
-
puts "No remaining gaps"
|
177
|
-
else
|
178
|
-
missing_ids.each do |id|
|
179
|
-
puts "Unable to find row with sequence ID #{id}"
|
180
|
-
end
|
181
|
-
end
|