aggregate_streams 0.0.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/lib/aggregate_streams.rb +15 -0
- data/lib/aggregate_streams/aggregate_streams.rb +35 -0
- data/lib/aggregate_streams/aggregation.rb +47 -0
- data/lib/aggregate_streams/consumer.rb +25 -0
- data/lib/aggregate_streams/controls.rb +13 -0
- data/lib/aggregate_streams/controls/aggregation.rb +33 -0
- data/lib/aggregate_streams/controls/category.rb +5 -0
- data/lib/aggregate_streams/controls/handler.rb +47 -0
- data/lib/aggregate_streams/controls/message_data.rb +39 -0
- data/lib/aggregate_streams/controls/message_data/metadata.rb +155 -0
- data/lib/aggregate_streams/controls/position.rb +51 -0
- data/lib/aggregate_streams/controls/store.rb +51 -0
- data/lib/aggregate_streams/controls/stream_name.rb +67 -0
- data/lib/aggregate_streams/handle.rb +148 -0
- data/lib/aggregate_streams/position_store.rb +51 -0
- data/lib/aggregate_streams/projection.rb +11 -0
- data/lib/aggregate_streams/store.rb +19 -0
- metadata +143 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 7c87c67f4a3dfcf93d0e5ce30380844638a58dc4edfc1ed04d259d19bffc50c7
|
4
|
+
data.tar.gz: ff1cbe61e2df7aa1da19f45202fcca08abe342f1ba1a9fa7b8d8cf9534762b23
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: d05878e3d60552a37e307504c86fbc52ebe468700ac36431128782baf4765d1f31666125d4bd7f1b90fc61bbb6603ce062fe3c690fd87b6f619e89372a8b4251
|
7
|
+
data.tar.gz: 0546551ec19669f9edbddb0e496a4424c08d3046b14037491daddfef08231256e8d6e644fec3f0498474fe4d934d1c8857bcbb2e28cc978bdb9d393fe9fe2f41
|
@@ -0,0 +1,15 @@
|
|
1
|
+
require 'consumer/postgres'
|
2
|
+
require 'entity_store'
|
3
|
+
require 'entity_snapshot/postgres'
|
4
|
+
require 'try'
|
5
|
+
|
6
|
+
require 'aggregate_streams/aggregation'
|
7
|
+
require 'aggregate_streams/projection'
|
8
|
+
require 'aggregate_streams/store'
|
9
|
+
require 'aggregate_streams/handle'
|
10
|
+
|
11
|
+
require 'aggregate_streams/position_store'
|
12
|
+
|
13
|
+
require 'aggregate_streams/consumer'
|
14
|
+
|
15
|
+
require 'aggregate_streams/aggregate_streams'
|
@@ -0,0 +1,35 @@
|
|
1
|
+
module AggregateStreams
|
2
|
+
def self.start(input_categories, output_category, writer_session: nil, snapshot_interval: nil, **consumer_args, &transform_action)
|
3
|
+
handler_block ||= proc { }
|
4
|
+
|
5
|
+
input_categories.each do |input_category|
|
6
|
+
handler_cls = Class.new do
|
7
|
+
include Handle
|
8
|
+
|
9
|
+
category output_category
|
10
|
+
|
11
|
+
unless writer_session.nil?
|
12
|
+
writer_session_macro do
|
13
|
+
writer_session
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
unless snapshot_interval.nil?
|
18
|
+
snapshot_interval_macro snapshot_interval
|
19
|
+
end
|
20
|
+
|
21
|
+
unless transform_action.nil?
|
22
|
+
transform(&transform_action)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
consumer_cls = Class.new do
|
27
|
+
include Consumer
|
28
|
+
|
29
|
+
handler handler_cls
|
30
|
+
end
|
31
|
+
|
32
|
+
consumer_cls.start(input_category, output_category: output_category, **consumer_args)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
module AggregateStreams
|
2
|
+
class Aggregation
|
3
|
+
include Schema::DataStructure
|
4
|
+
|
5
|
+
attribute :sequences, Hash, default: ->{ Hash.new }
|
6
|
+
|
7
|
+
def set_sequence(category, sequence)
|
8
|
+
sequences[category] = sequence
|
9
|
+
end
|
10
|
+
|
11
|
+
def sequence(category)
|
12
|
+
sequences[category]
|
13
|
+
end
|
14
|
+
|
15
|
+
def record_processed(message)
|
16
|
+
causation_stream_name = message.metadata.fetch(:causation_message_stream_name)
|
17
|
+
causation_global_position = message.metadata.fetch(:causation_message_global_position)
|
18
|
+
|
19
|
+
causation_category = Messaging::StreamName.get_category(causation_stream_name)
|
20
|
+
|
21
|
+
set_sequence(causation_category, causation_global_position)
|
22
|
+
end
|
23
|
+
|
24
|
+
def processed?(message)
|
25
|
+
message_category = Messaging::StreamName.get_category(message.stream_name)
|
26
|
+
|
27
|
+
sequence = sequence(message_category)
|
28
|
+
return false if sequence.nil?
|
29
|
+
|
30
|
+
sequence >= message.global_position
|
31
|
+
end
|
32
|
+
|
33
|
+
module Transform
|
34
|
+
def self.raw_data(aggregation)
|
35
|
+
aggregation.to_h
|
36
|
+
end
|
37
|
+
|
38
|
+
def self.instance(raw_data)
|
39
|
+
sequences = Casing::Camel.(raw_data[:sequences], symbol_to_string: true)
|
40
|
+
|
41
|
+
Aggregation.build({
|
42
|
+
:sequences => sequences
|
43
|
+
})
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
module AggregateStreams
|
2
|
+
module Consumer
|
3
|
+
def self.included(cls)
|
4
|
+
cls.class_exec do
|
5
|
+
include ::Consumer::Postgres
|
6
|
+
include Configure
|
7
|
+
end
|
8
|
+
end
|
9
|
+
|
10
|
+
module Configure
|
11
|
+
def configure(output_category:, output_session: nil, **args)
|
12
|
+
super(**args)
|
13
|
+
|
14
|
+
input_category = self.category
|
15
|
+
|
16
|
+
PositionStore.configure(
|
17
|
+
self,
|
18
|
+
input_category,
|
19
|
+
output_category,
|
20
|
+
session: session
|
21
|
+
)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
require 'messaging/controls'
|
2
|
+
|
3
|
+
require 'aggregate_streams/controls/message_data'
|
4
|
+
require 'aggregate_streams/controls/message_data/metadata'
|
5
|
+
require 'aggregate_streams/controls/position'
|
6
|
+
require 'aggregate_streams/controls/stream_name'
|
7
|
+
require 'aggregate_streams/controls/category'
|
8
|
+
|
9
|
+
require 'aggregate_streams/controls/aggregation'
|
10
|
+
|
11
|
+
require 'aggregate_streams/controls/store'
|
12
|
+
|
13
|
+
require 'aggregate_streams/controls/handler'
|
@@ -0,0 +1,33 @@
|
|
1
|
+
module AggregateStreams
|
2
|
+
module Controls
|
3
|
+
module Aggregation
|
4
|
+
def self.example(sequence: nil, sequence_category: nil)
|
5
|
+
if sequence == :none
|
6
|
+
sequence = nil
|
7
|
+
else
|
8
|
+
sequence ||= self.sequence
|
9
|
+
end
|
10
|
+
|
11
|
+
sequence_category ||= StreamName::Input.category
|
12
|
+
|
13
|
+
aggregation = New.example
|
14
|
+
|
15
|
+
unless sequence.nil?
|
16
|
+
aggregation.set_sequence(sequence_category, sequence)
|
17
|
+
end
|
18
|
+
|
19
|
+
aggregation
|
20
|
+
end
|
21
|
+
|
22
|
+
def self.sequence
|
23
|
+
Position.example
|
24
|
+
end
|
25
|
+
|
26
|
+
module New
|
27
|
+
def self.example
|
28
|
+
AggregateStreams::Aggregation.new
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
module AggregateStreams
|
2
|
+
module Controls
|
3
|
+
module Handler
|
4
|
+
def self.example(category: nil, snapshot: nil, snapshot_interval: nil, &specialize)
|
5
|
+
if category.nil? && snapshot.nil? && snapshot_interval.nil? && specialize.nil?
|
6
|
+
cls = Example
|
7
|
+
else
|
8
|
+
cls = example_class(category: category, snapshot: snapshot, snapshot_interval: snapshot_interval, &specialize)
|
9
|
+
end
|
10
|
+
|
11
|
+
cls.build
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.example_class(category: nil, snapshot: nil, snapshot_interval: nil, &specialize)
|
15
|
+
if category == :none
|
16
|
+
category = nil
|
17
|
+
else
|
18
|
+
category ||= Category.example
|
19
|
+
end
|
20
|
+
|
21
|
+
snapshot ||= false
|
22
|
+
|
23
|
+
if snapshot
|
24
|
+
snapshot_interval ||= Store.snapshot_interval
|
25
|
+
end
|
26
|
+
|
27
|
+
Class.new do
|
28
|
+
include AggregateStreams::Handle
|
29
|
+
|
30
|
+
unless category.nil?
|
31
|
+
category category
|
32
|
+
end
|
33
|
+
|
34
|
+
unless snapshot_interval.nil?
|
35
|
+
snapshot_interval snapshot_interval
|
36
|
+
end
|
37
|
+
|
38
|
+
unless specialize.nil?
|
39
|
+
class_exec(&specialize)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
Example = example_class
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
module AggregateStreams
|
2
|
+
module Controls
|
3
|
+
MessageData = MessageStore::Controls::MessageData
|
4
|
+
|
5
|
+
module MessageData
|
6
|
+
module Input
|
7
|
+
def self.example(type: nil, data: nil, metadata: nil, **metadata_args)
|
8
|
+
metadata ||= Metadata::Input.example(**metadata_args)
|
9
|
+
|
10
|
+
message_data = Read.example(type: type, data: data, metadata: metadata)
|
11
|
+
message_data.stream_name = metadata[:stream_name]
|
12
|
+
message_data.position = metadata[:position]
|
13
|
+
message_data.global_position = metadata[:global_position]
|
14
|
+
message_data
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.alternate(type: nil, data: nil)
|
18
|
+
metadata = Metadata::Input.alternate
|
19
|
+
|
20
|
+
Read.example(type: type, data: data, metadata: metadata)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
module Output
|
25
|
+
def self.example(type: nil, data: nil, metadata: nil, **metadata_args)
|
26
|
+
metadata ||= Metadata::Output.example(**metadata_args)
|
27
|
+
|
28
|
+
MessageData::Read.example(type: type, data: data, metadata: metadata)
|
29
|
+
end
|
30
|
+
|
31
|
+
def self.alternate(type: nil, data: nil)
|
32
|
+
metadata ||= Metadata::Output.alternate
|
33
|
+
|
34
|
+
MessageData::Read.example(type: type, data: data, metadata: metadata)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,155 @@
|
|
1
|
+
module AggregateStreams
|
2
|
+
module Controls
|
3
|
+
module MessageData
|
4
|
+
module Metadata
|
5
|
+
def self.example(category: nil, stream_id: nil, stream_name: nil, position: nil, global_position: nil, causation_message_stream_name: nil, causation_message_position: nil, causation_message_global_position: nil, correlation_stream_name: nil, reply_stream_name: nil, schema_version: nil)
|
6
|
+
if stream_name == :none
|
7
|
+
stream_name = nil
|
8
|
+
else
|
9
|
+
stream_name ||= stream_name(id: stream_id, category: category)
|
10
|
+
end
|
11
|
+
|
12
|
+
if position == :none
|
13
|
+
position = nil
|
14
|
+
else
|
15
|
+
position ||= self.position
|
16
|
+
end
|
17
|
+
|
18
|
+
if global_position == :none
|
19
|
+
global_position = nil
|
20
|
+
else
|
21
|
+
global_position ||= self.global_position
|
22
|
+
end
|
23
|
+
|
24
|
+
if causation_message_stream_name == :none
|
25
|
+
causation_message_stream_name = nil
|
26
|
+
else
|
27
|
+
causation_message_stream_name ||= self.causation_message_stream_name
|
28
|
+
end
|
29
|
+
|
30
|
+
if causation_message_position == :none
|
31
|
+
causation_message_position = nil
|
32
|
+
else
|
33
|
+
causation_message_position ||= self.causation_message_position
|
34
|
+
end
|
35
|
+
|
36
|
+
if causation_message_global_position == :none
|
37
|
+
causation_message_global_position = nil
|
38
|
+
else
|
39
|
+
causation_message_global_position ||= self.causation_message_global_position
|
40
|
+
end
|
41
|
+
|
42
|
+
if correlation_stream_name == :none
|
43
|
+
correlation_stream_name = nil
|
44
|
+
else
|
45
|
+
correlation_stream_name ||= Metadata.correlation_stream_name
|
46
|
+
end
|
47
|
+
|
48
|
+
if reply_stream_name == :none
|
49
|
+
reply_stream_name = nil
|
50
|
+
else
|
51
|
+
reply_stream_name ||= Metadata.reply_stream_name
|
52
|
+
end
|
53
|
+
|
54
|
+
if schema_version == :none
|
55
|
+
schema_version = nil
|
56
|
+
else
|
57
|
+
schema_version ||= Metadata.schema_version
|
58
|
+
end
|
59
|
+
|
60
|
+
metadata = {
|
61
|
+
:stream_name => stream_name,
|
62
|
+
:position => position,
|
63
|
+
:global_position => global_position,
|
64
|
+
|
65
|
+
:causation_message_stream_name => causation_message_stream_name,
|
66
|
+
:causation_message_position => causation_message_position,
|
67
|
+
:causation_message_global_position => causation_message_global_position,
|
68
|
+
|
69
|
+
:correlation_stream_name => correlation_stream_name,
|
70
|
+
:reply_stream_name => reply_stream_name,
|
71
|
+
:schema_version => schema_version
|
72
|
+
}
|
73
|
+
|
74
|
+
metadata.delete_if { |_, v| v.nil? }
|
75
|
+
|
76
|
+
metadata
|
77
|
+
end
|
78
|
+
|
79
|
+
def self.stream_name(**args)
|
80
|
+
StreamName.example(**args)
|
81
|
+
end
|
82
|
+
|
83
|
+
def self.position
|
84
|
+
Position.example
|
85
|
+
end
|
86
|
+
|
87
|
+
def self.global_position
|
88
|
+
Position::Global.example
|
89
|
+
end
|
90
|
+
|
91
|
+
def self.causation_message_stream_name
|
92
|
+
Messaging::Controls::Metadata.causation_message_stream_name
|
93
|
+
end
|
94
|
+
|
95
|
+
def self.causation_message_position
|
96
|
+
Messaging::Controls::Metadata.causation_message_position
|
97
|
+
end
|
98
|
+
|
99
|
+
def self.causation_message_global_position
|
100
|
+
Messaging::Controls::Metadata.causation_message_global_position
|
101
|
+
end
|
102
|
+
|
103
|
+
def self.correlation_stream_name
|
104
|
+
Messaging::Controls::Metadata.correlation_stream_name
|
105
|
+
end
|
106
|
+
|
107
|
+
def self.reply_stream_name
|
108
|
+
Messaging::Controls::Metadata.reply_stream_name
|
109
|
+
end
|
110
|
+
|
111
|
+
def self.schema_version
|
112
|
+
Messaging::Controls::Metadata.schema_version
|
113
|
+
end
|
114
|
+
|
115
|
+
module Input
|
116
|
+
def self.example
|
117
|
+
Metadata.example(
|
118
|
+
stream_name: StreamName::Input.example,
|
119
|
+
position: Position::Previous.example,
|
120
|
+
global_position: Position::Global::Previous.example
|
121
|
+
)
|
122
|
+
end
|
123
|
+
|
124
|
+
def self.alternate
|
125
|
+
Metadata.example(
|
126
|
+
stream_name: StreamName::Input::Alternate.example,
|
127
|
+
position: Position::Previous.alternate,
|
128
|
+
global_position: Position::Global::Previous.alternate
|
129
|
+
)
|
130
|
+
end
|
131
|
+
end
|
132
|
+
|
133
|
+
module Output
|
134
|
+
def self.example(causation_message_category: nil)
|
135
|
+
causation_message_category ||= StreamName::Input::Category.example
|
136
|
+
|
137
|
+
Metadata.example(
|
138
|
+
causation_message_stream_name: StreamName::Input.example(category: causation_message_category),
|
139
|
+
causation_message_position: Position::Previous.example,
|
140
|
+
causation_message_global_position: Position::Global::Previous.example
|
141
|
+
)
|
142
|
+
end
|
143
|
+
|
144
|
+
def self.alternate
|
145
|
+
Metadata.example(
|
146
|
+
causation_message_stream_name: StreamName::Input::Alternate.example,
|
147
|
+
causation_message_position: Position::Previous.alternate,
|
148
|
+
causation_message_global_position: Position::Global::Previous.alternate
|
149
|
+
)
|
150
|
+
end
|
151
|
+
end
|
152
|
+
end
|
153
|
+
end
|
154
|
+
end
|
155
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
module AggregateStreams
|
2
|
+
module Controls
|
3
|
+
module Position
|
4
|
+
def self.example
|
5
|
+
11
|
6
|
+
end
|
7
|
+
|
8
|
+
def self.alternate
|
9
|
+
22
|
10
|
+
end
|
11
|
+
|
12
|
+
module Previous
|
13
|
+
def self.example
|
14
|
+
Position.example - 1
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.alternate
|
18
|
+
Position.alternate - 1
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
module Initial
|
23
|
+
def self.example
|
24
|
+
0
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
module Global
|
29
|
+
def self.example
|
30
|
+
111
|
31
|
+
end
|
32
|
+
|
33
|
+
def self.alternate
|
34
|
+
222
|
35
|
+
end
|
36
|
+
|
37
|
+
module Previous
|
38
|
+
def self.example
|
39
|
+
Global.example - 1
|
40
|
+
end
|
41
|
+
|
42
|
+
def self.alternate
|
43
|
+
Global.alternate - 1
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
Causation = Previous
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
module AggregateStreams
|
2
|
+
module Controls
|
3
|
+
module Store
|
4
|
+
def self.example(category: nil, snapshot: nil, snapshot_interval: nil)
|
5
|
+
if category.nil? && snapshot.nil?
|
6
|
+
cls = Example
|
7
|
+
else
|
8
|
+
cls = example_class(category: category, snapshot: snapshot, snapshot_interval: snapshot_interval)
|
9
|
+
end
|
10
|
+
|
11
|
+
cls.build
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.example_class(category: nil, snapshot: nil, snapshot_interval: nil)
|
15
|
+
if category == :none
|
16
|
+
category = nil
|
17
|
+
else
|
18
|
+
category ||= self.category
|
19
|
+
end
|
20
|
+
|
21
|
+
snapshot ||= false
|
22
|
+
|
23
|
+
if snapshot
|
24
|
+
snapshot_interval ||= self.snapshot_interval
|
25
|
+
end
|
26
|
+
|
27
|
+
Class.new do
|
28
|
+
include AggregateStreams::Store
|
29
|
+
|
30
|
+
unless category.nil?
|
31
|
+
category category
|
32
|
+
end
|
33
|
+
|
34
|
+
unless snapshot_interval.nil?
|
35
|
+
snapshot EntitySnapshot::Postgres, interval: snapshot_interval
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def self.category
|
41
|
+
StreamName::Output::Category.internal
|
42
|
+
end
|
43
|
+
|
44
|
+
def self.snapshot_interval
|
45
|
+
11
|
46
|
+
end
|
47
|
+
|
48
|
+
Example = self.example_class
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -0,0 +1,67 @@
|
|
1
|
+
module AggregateStreams
|
2
|
+
module Controls
|
3
|
+
module StreamName
|
4
|
+
def self.example(**args)
|
5
|
+
MessageStore::Controls::StreamName.example(**args)
|
6
|
+
end
|
7
|
+
|
8
|
+
def self.stream_name(category, id=nil, **args)
|
9
|
+
MessageStore::Controls::StreamName.stream_name(category, id, **args)
|
10
|
+
end
|
11
|
+
|
12
|
+
module Input
|
13
|
+
def self.example(category: nil, **args)
|
14
|
+
category ||= self.category
|
15
|
+
|
16
|
+
StreamName.example(category: category, **args)
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.category
|
20
|
+
'someInput'
|
21
|
+
end
|
22
|
+
|
23
|
+
module Category
|
24
|
+
def self.example
|
25
|
+
StreamName.stream_name(Input.category)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
module Alternate
|
30
|
+
def self.example(**args)
|
31
|
+
Input.example(category: category, **args)
|
32
|
+
end
|
33
|
+
|
34
|
+
def self.category
|
35
|
+
'otherInput'
|
36
|
+
end
|
37
|
+
|
38
|
+
module Category
|
39
|
+
def self.example
|
40
|
+
StreamName.stream_name(Alternate.category)
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
module Output
|
47
|
+
def self.example(**args)
|
48
|
+
StreamName.example(category: category, **args)
|
49
|
+
end
|
50
|
+
|
51
|
+
def self.category
|
52
|
+
'someAggregate'
|
53
|
+
end
|
54
|
+
|
55
|
+
module Category
|
56
|
+
def self.example
|
57
|
+
StreamName.stream_name(Output.category)
|
58
|
+
end
|
59
|
+
|
60
|
+
def self.internal
|
61
|
+
:some_aggregate
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
@@ -0,0 +1,148 @@
|
|
1
|
+
module AggregateStreams
|
2
|
+
module Handle
|
3
|
+
TransformError = Class.new(RuntimeError)
|
4
|
+
|
5
|
+
def self.included(cls)
|
6
|
+
cls.class_exec do
|
7
|
+
include Messaging::Handle
|
8
|
+
include Messaging::StreamName
|
9
|
+
|
10
|
+
include Log::Dependency
|
11
|
+
|
12
|
+
prepend Configure
|
13
|
+
|
14
|
+
extend StoreClass
|
15
|
+
extend CategoryMacro
|
16
|
+
extend TransformMacro
|
17
|
+
extend SnapshotIntervalMacro
|
18
|
+
extend WriterSessionMacro
|
19
|
+
|
20
|
+
const_set :Store, store_class
|
21
|
+
|
22
|
+
dependency :store, self::Store
|
23
|
+
dependency :write, MessageStore::Postgres::Write
|
24
|
+
|
25
|
+
virtual :writer_session
|
26
|
+
|
27
|
+
virtual :transform_action do
|
28
|
+
proc { |message_data| message_data }
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def handle(message_data)
|
34
|
+
logger.trace { "Handling message (Stream: #{message_data.stream_name}, Global Position: #{message_data.global_position})" }
|
35
|
+
|
36
|
+
stream_id = Messaging::StreamName.get_id(message_data.stream_name)
|
37
|
+
|
38
|
+
aggregation, version = store.fetch(stream_id, include: :version)
|
39
|
+
|
40
|
+
if aggregation.processed?(message_data)
|
41
|
+
logger.info(tag: :ignored) { "Message already handled (Stream: #{message_data.stream_name}, Global Position: #{message_data.global_position})" }
|
42
|
+
return
|
43
|
+
end
|
44
|
+
|
45
|
+
raw_input_data = Messaging::Message::Transform::MessageData.read(message_data)
|
46
|
+
input_metadata = Messaging::Message::Metadata.build(raw_input_data[:metadata])
|
47
|
+
|
48
|
+
output_metadata = Messaging::Message::Metadata.build
|
49
|
+
|
50
|
+
output_metadata.follow(input_metadata)
|
51
|
+
|
52
|
+
output_metadata = output_metadata.to_h
|
53
|
+
output_metadata.delete_if { |_, v| v.nil? }
|
54
|
+
|
55
|
+
write_message_data = MessageStore::MessageData::Write.new
|
56
|
+
|
57
|
+
SetAttributes.(write_message_data, message_data, copy: [:type, :data])
|
58
|
+
|
59
|
+
write_message_data.metadata = output_metadata
|
60
|
+
|
61
|
+
input_category = Messaging::StreamName.get_category(message_data.stream_name)
|
62
|
+
write_message_data = transform(write_message_data, input_category)
|
63
|
+
|
64
|
+
if write_message_data
|
65
|
+
assure_message_data(write_message_data)
|
66
|
+
else
|
67
|
+
logger.info(tag: :ignored) { "Message ignored (Stream: #{message_data.stream_name}, Global Position: #{message_data.global_position})" }
|
68
|
+
return
|
69
|
+
end
|
70
|
+
|
71
|
+
logger.info do
|
72
|
+
message_type = message_data.type
|
73
|
+
unless write_message_data.type == message_type
|
74
|
+
message_type = "#{write_message_data.type} ← #{message_type}"
|
75
|
+
end
|
76
|
+
|
77
|
+
"Message copied (Message Type: #{message_type}, Stream: #{message_data.stream_name}, Global Position: #{message_data.global_position})"
|
78
|
+
end
|
79
|
+
|
80
|
+
Try.(MessageStore::ExpectedVersion::Error) do
|
81
|
+
stream_name = stream_name(stream_id)
|
82
|
+
write.(write_message_data, stream_name, expected_version: version)
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
def transform(write_message_data, stream_name)
|
87
|
+
transform_action.(write_message_data, stream_name)
|
88
|
+
end
|
89
|
+
|
90
|
+
def assure_message_data(message_data)
|
91
|
+
unless message_data.instance_of?(MessageStore::MessageData::Write)
|
92
|
+
raise TransformError, "Not an instance of MessageData::Write"
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
module Configure
|
97
|
+
def configure(session: nil)
|
98
|
+
writer_session = self.writer_session
|
99
|
+
writer_session ||= session
|
100
|
+
|
101
|
+
self.class::Store.configure(self, session: writer_session)
|
102
|
+
|
103
|
+
MessageStore::Postgres::Write.configure(self, session: writer_session)
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
module StoreClass
|
108
|
+
def store_class
|
109
|
+
@store_class ||= Class.new do
|
110
|
+
include Store
|
111
|
+
end
|
112
|
+
end
|
113
|
+
alias_method :store_cls, :store_class
|
114
|
+
end
|
115
|
+
|
116
|
+
module CategoryMacro
|
117
|
+
def category_macro(category)
|
118
|
+
super(category)
|
119
|
+
|
120
|
+
store_class.category_macro(category)
|
121
|
+
end
|
122
|
+
alias_method :category, :category_macro
|
123
|
+
end
|
124
|
+
|
125
|
+
module TransformMacro
|
126
|
+
def transform_macro(&transform_action)
|
127
|
+
define_method(:transform_action) do
|
128
|
+
transform_action
|
129
|
+
end
|
130
|
+
end
|
131
|
+
alias_method :transform, :transform_macro
|
132
|
+
end
|
133
|
+
|
134
|
+
module SnapshotIntervalMacro
|
135
|
+
def snapshot_interval_macro(interval)
|
136
|
+
store_class.snapshot(EntitySnapshot::Postgres, interval: interval)
|
137
|
+
end
|
138
|
+
alias_method :snapshot_interval, :snapshot_interval_macro
|
139
|
+
end
|
140
|
+
|
141
|
+
module WriterSessionMacro
|
142
|
+
def writer_session_macro(&block)
|
143
|
+
define_method(:writer_session, &block)
|
144
|
+
end
|
145
|
+
alias_method :writer_session, :writer_session_macro
|
146
|
+
end
|
147
|
+
end
|
148
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
module AggregateStreams
|
2
|
+
class PositionStore
|
3
|
+
include ::Consumer::PositionStore
|
4
|
+
include ::Log::Dependency
|
5
|
+
include Initializer
|
6
|
+
|
7
|
+
dependency :session, MessageStore::Postgres::Session
|
8
|
+
|
9
|
+
initializer :input_category, :output_category
|
10
|
+
|
11
|
+
def self.build(input_category, output_category, session: nil)
|
12
|
+
instance = new(input_category, output_category)
|
13
|
+
MessageStore::Postgres::Session.configure(instance, session: session)
|
14
|
+
instance
|
15
|
+
end
|
16
|
+
|
17
|
+
def get
|
18
|
+
logger.trace { "Get position (Output Category: #{output_category.inspect}, Input Category: #{input_category.inspect})" }
|
19
|
+
|
20
|
+
sql_command = self.class.get_sql_command
|
21
|
+
|
22
|
+
parameter_values = [output_category, input_category]
|
23
|
+
|
24
|
+
result = session.execute(sql_command, parameter_values)
|
25
|
+
|
26
|
+
if result.ntuples.zero?
|
27
|
+
position = nil
|
28
|
+
else
|
29
|
+
record = result[0]
|
30
|
+
|
31
|
+
position = record['position'].to_i + 1
|
32
|
+
end
|
33
|
+
|
34
|
+
logger.info { "Get position done (Position: #{position || '(none)'}, Output Category: #{output_category.inspect}, Input Category: #{input_category.inspect})" }
|
35
|
+
|
36
|
+
position
|
37
|
+
end
|
38
|
+
|
39
|
+
def self.get_sql_command
|
40
|
+
%{
|
41
|
+
SELECT
|
42
|
+
metadata->>'causationMessageGlobalPosition' AS position
|
43
|
+
FROM messages
|
44
|
+
WHERE
|
45
|
+
category(stream_name) = $1 AND
|
46
|
+
category(metadata->>'causationMessageStreamName') = $2
|
47
|
+
ORDER BY global_position DESC LIMIT 1
|
48
|
+
}
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
module AggregateStreams
|
2
|
+
module Store
|
3
|
+
def self.included(cls)
|
4
|
+
cls.class_exec do
|
5
|
+
include EntityStore
|
6
|
+
|
7
|
+
entity Aggregation
|
8
|
+
projection Projection
|
9
|
+
reader MessageStore::Postgres::Read, batch_size: Defaults.batch_size
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
module Defaults
|
14
|
+
def self.batch_size
|
15
|
+
MessageStore::Postgres::Read::Defaults.batch_size
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
metadata
ADDED
@@ -0,0 +1,143 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: aggregate_streams
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.0.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Nathan Ladd
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2020-11-09 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: evt-consumer-postgres
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: evt-entity_store
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: evt-entity_snapshot-postgres
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: evt-try
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: test_bench
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: evt-component_host
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
97
|
+
description: " "
|
98
|
+
email: nathanladd@gmail.com
|
99
|
+
executables: []
|
100
|
+
extensions: []
|
101
|
+
extra_rdoc_files: []
|
102
|
+
files:
|
103
|
+
- lib/aggregate_streams.rb
|
104
|
+
- lib/aggregate_streams/aggregate_streams.rb
|
105
|
+
- lib/aggregate_streams/aggregation.rb
|
106
|
+
- lib/aggregate_streams/consumer.rb
|
107
|
+
- lib/aggregate_streams/controls.rb
|
108
|
+
- lib/aggregate_streams/controls/aggregation.rb
|
109
|
+
- lib/aggregate_streams/controls/category.rb
|
110
|
+
- lib/aggregate_streams/controls/handler.rb
|
111
|
+
- lib/aggregate_streams/controls/message_data.rb
|
112
|
+
- lib/aggregate_streams/controls/message_data/metadata.rb
|
113
|
+
- lib/aggregate_streams/controls/position.rb
|
114
|
+
- lib/aggregate_streams/controls/store.rb
|
115
|
+
- lib/aggregate_streams/controls/stream_name.rb
|
116
|
+
- lib/aggregate_streams/handle.rb
|
117
|
+
- lib/aggregate_streams/position_store.rb
|
118
|
+
- lib/aggregate_streams/projection.rb
|
119
|
+
- lib/aggregate_streams/store.rb
|
120
|
+
homepage: https://github.com/ntl/aggregate-streams
|
121
|
+
licenses:
|
122
|
+
- MIT
|
123
|
+
metadata: {}
|
124
|
+
post_install_message:
|
125
|
+
rdoc_options: []
|
126
|
+
require_paths:
|
127
|
+
- lib
|
128
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
129
|
+
requirements:
|
130
|
+
- - ">="
|
131
|
+
- !ruby/object:Gem::Version
|
132
|
+
version: '2.7'
|
133
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
134
|
+
requirements:
|
135
|
+
- - ">="
|
136
|
+
- !ruby/object:Gem::Version
|
137
|
+
version: '0'
|
138
|
+
requirements: []
|
139
|
+
rubygems_version: 3.1.4
|
140
|
+
signing_key:
|
141
|
+
specification_version: 4
|
142
|
+
summary: Combine messages from multiple Eventide streams into an aggregate stream
|
143
|
+
test_files: []
|