aggregate_streams 0.0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/lib/aggregate_streams.rb +15 -0
- data/lib/aggregate_streams/aggregate_streams.rb +35 -0
- data/lib/aggregate_streams/aggregation.rb +47 -0
- data/lib/aggregate_streams/consumer.rb +25 -0
- data/lib/aggregate_streams/controls.rb +13 -0
- data/lib/aggregate_streams/controls/aggregation.rb +33 -0
- data/lib/aggregate_streams/controls/category.rb +5 -0
- data/lib/aggregate_streams/controls/handler.rb +47 -0
- data/lib/aggregate_streams/controls/message_data.rb +39 -0
- data/lib/aggregate_streams/controls/message_data/metadata.rb +155 -0
- data/lib/aggregate_streams/controls/position.rb +51 -0
- data/lib/aggregate_streams/controls/store.rb +51 -0
- data/lib/aggregate_streams/controls/stream_name.rb +67 -0
- data/lib/aggregate_streams/handle.rb +148 -0
- data/lib/aggregate_streams/position_store.rb +51 -0
- data/lib/aggregate_streams/projection.rb +11 -0
- data/lib/aggregate_streams/store.rb +19 -0
- metadata +143 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 7c87c67f4a3dfcf93d0e5ce30380844638a58dc4edfc1ed04d259d19bffc50c7
|
4
|
+
data.tar.gz: ff1cbe61e2df7aa1da19f45202fcca08abe342f1ba1a9fa7b8d8cf9534762b23
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: d05878e3d60552a37e307504c86fbc52ebe468700ac36431128782baf4765d1f31666125d4bd7f1b90fc61bbb6603ce062fe3c690fd87b6f619e89372a8b4251
|
7
|
+
data.tar.gz: 0546551ec19669f9edbddb0e496a4424c08d3046b14037491daddfef08231256e8d6e644fec3f0498474fe4d934d1c8857bcbb2e28cc978bdb9d393fe9fe2f41
|
@@ -0,0 +1,15 @@
|
|
1
|
+
require 'consumer/postgres'
|
2
|
+
require 'entity_store'
|
3
|
+
require 'entity_snapshot/postgres'
|
4
|
+
require 'try'
|
5
|
+
|
6
|
+
require 'aggregate_streams/aggregation'
|
7
|
+
require 'aggregate_streams/projection'
|
8
|
+
require 'aggregate_streams/store'
|
9
|
+
require 'aggregate_streams/handle'
|
10
|
+
|
11
|
+
require 'aggregate_streams/position_store'
|
12
|
+
|
13
|
+
require 'aggregate_streams/consumer'
|
14
|
+
|
15
|
+
require 'aggregate_streams/aggregate_streams'
|
@@ -0,0 +1,35 @@
|
|
1
|
+
module AggregateStreams
|
2
|
+
def self.start(input_categories, output_category, writer_session: nil, snapshot_interval: nil, **consumer_args, &transform_action)
|
3
|
+
handler_block ||= proc { }
|
4
|
+
|
5
|
+
input_categories.each do |input_category|
|
6
|
+
handler_cls = Class.new do
|
7
|
+
include Handle
|
8
|
+
|
9
|
+
category output_category
|
10
|
+
|
11
|
+
unless writer_session.nil?
|
12
|
+
writer_session_macro do
|
13
|
+
writer_session
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
unless snapshot_interval.nil?
|
18
|
+
snapshot_interval_macro snapshot_interval
|
19
|
+
end
|
20
|
+
|
21
|
+
unless transform_action.nil?
|
22
|
+
transform(&transform_action)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
consumer_cls = Class.new do
|
27
|
+
include Consumer
|
28
|
+
|
29
|
+
handler handler_cls
|
30
|
+
end
|
31
|
+
|
32
|
+
consumer_cls.start(input_category, output_category: output_category, **consumer_args)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
module AggregateStreams
|
2
|
+
class Aggregation
|
3
|
+
include Schema::DataStructure
|
4
|
+
|
5
|
+
attribute :sequences, Hash, default: ->{ Hash.new }
|
6
|
+
|
7
|
+
def set_sequence(category, sequence)
|
8
|
+
sequences[category] = sequence
|
9
|
+
end
|
10
|
+
|
11
|
+
def sequence(category)
|
12
|
+
sequences[category]
|
13
|
+
end
|
14
|
+
|
15
|
+
def record_processed(message)
|
16
|
+
causation_stream_name = message.metadata.fetch(:causation_message_stream_name)
|
17
|
+
causation_global_position = message.metadata.fetch(:causation_message_global_position)
|
18
|
+
|
19
|
+
causation_category = Messaging::StreamName.get_category(causation_stream_name)
|
20
|
+
|
21
|
+
set_sequence(causation_category, causation_global_position)
|
22
|
+
end
|
23
|
+
|
24
|
+
def processed?(message)
|
25
|
+
message_category = Messaging::StreamName.get_category(message.stream_name)
|
26
|
+
|
27
|
+
sequence = sequence(message_category)
|
28
|
+
return false if sequence.nil?
|
29
|
+
|
30
|
+
sequence >= message.global_position
|
31
|
+
end
|
32
|
+
|
33
|
+
module Transform
|
34
|
+
def self.raw_data(aggregation)
|
35
|
+
aggregation.to_h
|
36
|
+
end
|
37
|
+
|
38
|
+
def self.instance(raw_data)
|
39
|
+
sequences = Casing::Camel.(raw_data[:sequences], symbol_to_string: true)
|
40
|
+
|
41
|
+
Aggregation.build({
|
42
|
+
:sequences => sequences
|
43
|
+
})
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
module AggregateStreams
|
2
|
+
module Consumer
|
3
|
+
def self.included(cls)
|
4
|
+
cls.class_exec do
|
5
|
+
include ::Consumer::Postgres
|
6
|
+
include Configure
|
7
|
+
end
|
8
|
+
end
|
9
|
+
|
10
|
+
module Configure
|
11
|
+
def configure(output_category:, output_session: nil, **args)
|
12
|
+
super(**args)
|
13
|
+
|
14
|
+
input_category = self.category
|
15
|
+
|
16
|
+
PositionStore.configure(
|
17
|
+
self,
|
18
|
+
input_category,
|
19
|
+
output_category,
|
20
|
+
session: session
|
21
|
+
)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
require 'messaging/controls'
|
2
|
+
|
3
|
+
require 'aggregate_streams/controls/message_data'
|
4
|
+
require 'aggregate_streams/controls/message_data/metadata'
|
5
|
+
require 'aggregate_streams/controls/position'
|
6
|
+
require 'aggregate_streams/controls/stream_name'
|
7
|
+
require 'aggregate_streams/controls/category'
|
8
|
+
|
9
|
+
require 'aggregate_streams/controls/aggregation'
|
10
|
+
|
11
|
+
require 'aggregate_streams/controls/store'
|
12
|
+
|
13
|
+
require 'aggregate_streams/controls/handler'
|
@@ -0,0 +1,33 @@
|
|
1
|
+
module AggregateStreams
|
2
|
+
module Controls
|
3
|
+
module Aggregation
|
4
|
+
def self.example(sequence: nil, sequence_category: nil)
|
5
|
+
if sequence == :none
|
6
|
+
sequence = nil
|
7
|
+
else
|
8
|
+
sequence ||= self.sequence
|
9
|
+
end
|
10
|
+
|
11
|
+
sequence_category ||= StreamName::Input.category
|
12
|
+
|
13
|
+
aggregation = New.example
|
14
|
+
|
15
|
+
unless sequence.nil?
|
16
|
+
aggregation.set_sequence(sequence_category, sequence)
|
17
|
+
end
|
18
|
+
|
19
|
+
aggregation
|
20
|
+
end
|
21
|
+
|
22
|
+
def self.sequence
|
23
|
+
Position.example
|
24
|
+
end
|
25
|
+
|
26
|
+
module New
|
27
|
+
def self.example
|
28
|
+
AggregateStreams::Aggregation.new
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
module AggregateStreams
|
2
|
+
module Controls
|
3
|
+
module Handler
|
4
|
+
def self.example(category: nil, snapshot: nil, snapshot_interval: nil, &specialize)
|
5
|
+
if category.nil? && snapshot.nil? && snapshot_interval.nil? && specialize.nil?
|
6
|
+
cls = Example
|
7
|
+
else
|
8
|
+
cls = example_class(category: category, snapshot: snapshot, snapshot_interval: snapshot_interval, &specialize)
|
9
|
+
end
|
10
|
+
|
11
|
+
cls.build
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.example_class(category: nil, snapshot: nil, snapshot_interval: nil, &specialize)
|
15
|
+
if category == :none
|
16
|
+
category = nil
|
17
|
+
else
|
18
|
+
category ||= Category.example
|
19
|
+
end
|
20
|
+
|
21
|
+
snapshot ||= false
|
22
|
+
|
23
|
+
if snapshot
|
24
|
+
snapshot_interval ||= Store.snapshot_interval
|
25
|
+
end
|
26
|
+
|
27
|
+
Class.new do
|
28
|
+
include AggregateStreams::Handle
|
29
|
+
|
30
|
+
unless category.nil?
|
31
|
+
category category
|
32
|
+
end
|
33
|
+
|
34
|
+
unless snapshot_interval.nil?
|
35
|
+
snapshot_interval snapshot_interval
|
36
|
+
end
|
37
|
+
|
38
|
+
unless specialize.nil?
|
39
|
+
class_exec(&specialize)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
Example = example_class
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
module AggregateStreams
|
2
|
+
module Controls
|
3
|
+
MessageData = MessageStore::Controls::MessageData
|
4
|
+
|
5
|
+
module MessageData
|
6
|
+
module Input
|
7
|
+
def self.example(type: nil, data: nil, metadata: nil, **metadata_args)
|
8
|
+
metadata ||= Metadata::Input.example(**metadata_args)
|
9
|
+
|
10
|
+
message_data = Read.example(type: type, data: data, metadata: metadata)
|
11
|
+
message_data.stream_name = metadata[:stream_name]
|
12
|
+
message_data.position = metadata[:position]
|
13
|
+
message_data.global_position = metadata[:global_position]
|
14
|
+
message_data
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.alternate(type: nil, data: nil)
|
18
|
+
metadata = Metadata::Input.alternate
|
19
|
+
|
20
|
+
Read.example(type: type, data: data, metadata: metadata)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
module Output
|
25
|
+
def self.example(type: nil, data: nil, metadata: nil, **metadata_args)
|
26
|
+
metadata ||= Metadata::Output.example(**metadata_args)
|
27
|
+
|
28
|
+
MessageData::Read.example(type: type, data: data, metadata: metadata)
|
29
|
+
end
|
30
|
+
|
31
|
+
def self.alternate(type: nil, data: nil)
|
32
|
+
metadata ||= Metadata::Output.alternate
|
33
|
+
|
34
|
+
MessageData::Read.example(type: type, data: data, metadata: metadata)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,155 @@
|
|
1
|
+
module AggregateStreams
|
2
|
+
module Controls
|
3
|
+
module MessageData
|
4
|
+
module Metadata
|
5
|
+
def self.example(category: nil, stream_id: nil, stream_name: nil, position: nil, global_position: nil, causation_message_stream_name: nil, causation_message_position: nil, causation_message_global_position: nil, correlation_stream_name: nil, reply_stream_name: nil, schema_version: nil)
|
6
|
+
if stream_name == :none
|
7
|
+
stream_name = nil
|
8
|
+
else
|
9
|
+
stream_name ||= stream_name(id: stream_id, category: category)
|
10
|
+
end
|
11
|
+
|
12
|
+
if position == :none
|
13
|
+
position = nil
|
14
|
+
else
|
15
|
+
position ||= self.position
|
16
|
+
end
|
17
|
+
|
18
|
+
if global_position == :none
|
19
|
+
global_position = nil
|
20
|
+
else
|
21
|
+
global_position ||= self.global_position
|
22
|
+
end
|
23
|
+
|
24
|
+
if causation_message_stream_name == :none
|
25
|
+
causation_message_stream_name = nil
|
26
|
+
else
|
27
|
+
causation_message_stream_name ||= self.causation_message_stream_name
|
28
|
+
end
|
29
|
+
|
30
|
+
if causation_message_position == :none
|
31
|
+
causation_message_position = nil
|
32
|
+
else
|
33
|
+
causation_message_position ||= self.causation_message_position
|
34
|
+
end
|
35
|
+
|
36
|
+
if causation_message_global_position == :none
|
37
|
+
causation_message_global_position = nil
|
38
|
+
else
|
39
|
+
causation_message_global_position ||= self.causation_message_global_position
|
40
|
+
end
|
41
|
+
|
42
|
+
if correlation_stream_name == :none
|
43
|
+
correlation_stream_name = nil
|
44
|
+
else
|
45
|
+
correlation_stream_name ||= Metadata.correlation_stream_name
|
46
|
+
end
|
47
|
+
|
48
|
+
if reply_stream_name == :none
|
49
|
+
reply_stream_name = nil
|
50
|
+
else
|
51
|
+
reply_stream_name ||= Metadata.reply_stream_name
|
52
|
+
end
|
53
|
+
|
54
|
+
if schema_version == :none
|
55
|
+
schema_version = nil
|
56
|
+
else
|
57
|
+
schema_version ||= Metadata.schema_version
|
58
|
+
end
|
59
|
+
|
60
|
+
metadata = {
|
61
|
+
:stream_name => stream_name,
|
62
|
+
:position => position,
|
63
|
+
:global_position => global_position,
|
64
|
+
|
65
|
+
:causation_message_stream_name => causation_message_stream_name,
|
66
|
+
:causation_message_position => causation_message_position,
|
67
|
+
:causation_message_global_position => causation_message_global_position,
|
68
|
+
|
69
|
+
:correlation_stream_name => correlation_stream_name,
|
70
|
+
:reply_stream_name => reply_stream_name,
|
71
|
+
:schema_version => schema_version
|
72
|
+
}
|
73
|
+
|
74
|
+
metadata.delete_if { |_, v| v.nil? }
|
75
|
+
|
76
|
+
metadata
|
77
|
+
end
|
78
|
+
|
79
|
+
def self.stream_name(**args)
|
80
|
+
StreamName.example(**args)
|
81
|
+
end
|
82
|
+
|
83
|
+
def self.position
|
84
|
+
Position.example
|
85
|
+
end
|
86
|
+
|
87
|
+
def self.global_position
|
88
|
+
Position::Global.example
|
89
|
+
end
|
90
|
+
|
91
|
+
def self.causation_message_stream_name
|
92
|
+
Messaging::Controls::Metadata.causation_message_stream_name
|
93
|
+
end
|
94
|
+
|
95
|
+
def self.causation_message_position
|
96
|
+
Messaging::Controls::Metadata.causation_message_position
|
97
|
+
end
|
98
|
+
|
99
|
+
def self.causation_message_global_position
|
100
|
+
Messaging::Controls::Metadata.causation_message_global_position
|
101
|
+
end
|
102
|
+
|
103
|
+
def self.correlation_stream_name
|
104
|
+
Messaging::Controls::Metadata.correlation_stream_name
|
105
|
+
end
|
106
|
+
|
107
|
+
def self.reply_stream_name
|
108
|
+
Messaging::Controls::Metadata.reply_stream_name
|
109
|
+
end
|
110
|
+
|
111
|
+
def self.schema_version
|
112
|
+
Messaging::Controls::Metadata.schema_version
|
113
|
+
end
|
114
|
+
|
115
|
+
module Input
|
116
|
+
def self.example
|
117
|
+
Metadata.example(
|
118
|
+
stream_name: StreamName::Input.example,
|
119
|
+
position: Position::Previous.example,
|
120
|
+
global_position: Position::Global::Previous.example
|
121
|
+
)
|
122
|
+
end
|
123
|
+
|
124
|
+
def self.alternate
|
125
|
+
Metadata.example(
|
126
|
+
stream_name: StreamName::Input::Alternate.example,
|
127
|
+
position: Position::Previous.alternate,
|
128
|
+
global_position: Position::Global::Previous.alternate
|
129
|
+
)
|
130
|
+
end
|
131
|
+
end
|
132
|
+
|
133
|
+
module Output
|
134
|
+
def self.example(causation_message_category: nil)
|
135
|
+
causation_message_category ||= StreamName::Input::Category.example
|
136
|
+
|
137
|
+
Metadata.example(
|
138
|
+
causation_message_stream_name: StreamName::Input.example(category: causation_message_category),
|
139
|
+
causation_message_position: Position::Previous.example,
|
140
|
+
causation_message_global_position: Position::Global::Previous.example
|
141
|
+
)
|
142
|
+
end
|
143
|
+
|
144
|
+
def self.alternate
|
145
|
+
Metadata.example(
|
146
|
+
causation_message_stream_name: StreamName::Input::Alternate.example,
|
147
|
+
causation_message_position: Position::Previous.alternate,
|
148
|
+
causation_message_global_position: Position::Global::Previous.alternate
|
149
|
+
)
|
150
|
+
end
|
151
|
+
end
|
152
|
+
end
|
153
|
+
end
|
154
|
+
end
|
155
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
module AggregateStreams
|
2
|
+
module Controls
|
3
|
+
module Position
|
4
|
+
def self.example
|
5
|
+
11
|
6
|
+
end
|
7
|
+
|
8
|
+
def self.alternate
|
9
|
+
22
|
10
|
+
end
|
11
|
+
|
12
|
+
module Previous
|
13
|
+
def self.example
|
14
|
+
Position.example - 1
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.alternate
|
18
|
+
Position.alternate - 1
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
module Initial
|
23
|
+
def self.example
|
24
|
+
0
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
module Global
|
29
|
+
def self.example
|
30
|
+
111
|
31
|
+
end
|
32
|
+
|
33
|
+
def self.alternate
|
34
|
+
222
|
35
|
+
end
|
36
|
+
|
37
|
+
module Previous
|
38
|
+
def self.example
|
39
|
+
Global.example - 1
|
40
|
+
end
|
41
|
+
|
42
|
+
def self.alternate
|
43
|
+
Global.alternate - 1
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
Causation = Previous
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
module AggregateStreams
|
2
|
+
module Controls
|
3
|
+
module Store
|
4
|
+
def self.example(category: nil, snapshot: nil, snapshot_interval: nil)
|
5
|
+
if category.nil? && snapshot.nil?
|
6
|
+
cls = Example
|
7
|
+
else
|
8
|
+
cls = example_class(category: category, snapshot: snapshot, snapshot_interval: snapshot_interval)
|
9
|
+
end
|
10
|
+
|
11
|
+
cls.build
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.example_class(category: nil, snapshot: nil, snapshot_interval: nil)
|
15
|
+
if category == :none
|
16
|
+
category = nil
|
17
|
+
else
|
18
|
+
category ||= self.category
|
19
|
+
end
|
20
|
+
|
21
|
+
snapshot ||= false
|
22
|
+
|
23
|
+
if snapshot
|
24
|
+
snapshot_interval ||= self.snapshot_interval
|
25
|
+
end
|
26
|
+
|
27
|
+
Class.new do
|
28
|
+
include AggregateStreams::Store
|
29
|
+
|
30
|
+
unless category.nil?
|
31
|
+
category category
|
32
|
+
end
|
33
|
+
|
34
|
+
unless snapshot_interval.nil?
|
35
|
+
snapshot EntitySnapshot::Postgres, interval: snapshot_interval
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def self.category
|
41
|
+
StreamName::Output::Category.internal
|
42
|
+
end
|
43
|
+
|
44
|
+
def self.snapshot_interval
|
45
|
+
11
|
46
|
+
end
|
47
|
+
|
48
|
+
Example = self.example_class
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -0,0 +1,67 @@
|
|
1
|
+
module AggregateStreams
|
2
|
+
module Controls
|
3
|
+
module StreamName
|
4
|
+
def self.example(**args)
|
5
|
+
MessageStore::Controls::StreamName.example(**args)
|
6
|
+
end
|
7
|
+
|
8
|
+
def self.stream_name(category, id=nil, **args)
|
9
|
+
MessageStore::Controls::StreamName.stream_name(category, id, **args)
|
10
|
+
end
|
11
|
+
|
12
|
+
module Input
|
13
|
+
def self.example(category: nil, **args)
|
14
|
+
category ||= self.category
|
15
|
+
|
16
|
+
StreamName.example(category: category, **args)
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.category
|
20
|
+
'someInput'
|
21
|
+
end
|
22
|
+
|
23
|
+
module Category
|
24
|
+
def self.example
|
25
|
+
StreamName.stream_name(Input.category)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
module Alternate
|
30
|
+
def self.example(**args)
|
31
|
+
Input.example(category: category, **args)
|
32
|
+
end
|
33
|
+
|
34
|
+
def self.category
|
35
|
+
'otherInput'
|
36
|
+
end
|
37
|
+
|
38
|
+
module Category
|
39
|
+
def self.example
|
40
|
+
StreamName.stream_name(Alternate.category)
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
module Output
|
47
|
+
def self.example(**args)
|
48
|
+
StreamName.example(category: category, **args)
|
49
|
+
end
|
50
|
+
|
51
|
+
def self.category
|
52
|
+
'someAggregate'
|
53
|
+
end
|
54
|
+
|
55
|
+
module Category
|
56
|
+
def self.example
|
57
|
+
StreamName.stream_name(Output.category)
|
58
|
+
end
|
59
|
+
|
60
|
+
def self.internal
|
61
|
+
:some_aggregate
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
@@ -0,0 +1,148 @@
|
|
1
|
+
module AggregateStreams
|
2
|
+
module Handle
|
3
|
+
TransformError = Class.new(RuntimeError)
|
4
|
+
|
5
|
+
def self.included(cls)
|
6
|
+
cls.class_exec do
|
7
|
+
include Messaging::Handle
|
8
|
+
include Messaging::StreamName
|
9
|
+
|
10
|
+
include Log::Dependency
|
11
|
+
|
12
|
+
prepend Configure
|
13
|
+
|
14
|
+
extend StoreClass
|
15
|
+
extend CategoryMacro
|
16
|
+
extend TransformMacro
|
17
|
+
extend SnapshotIntervalMacro
|
18
|
+
extend WriterSessionMacro
|
19
|
+
|
20
|
+
const_set :Store, store_class
|
21
|
+
|
22
|
+
dependency :store, self::Store
|
23
|
+
dependency :write, MessageStore::Postgres::Write
|
24
|
+
|
25
|
+
virtual :writer_session
|
26
|
+
|
27
|
+
virtual :transform_action do
|
28
|
+
proc { |message_data| message_data }
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def handle(message_data)
|
34
|
+
logger.trace { "Handling message (Stream: #{message_data.stream_name}, Global Position: #{message_data.global_position})" }
|
35
|
+
|
36
|
+
stream_id = Messaging::StreamName.get_id(message_data.stream_name)
|
37
|
+
|
38
|
+
aggregation, version = store.fetch(stream_id, include: :version)
|
39
|
+
|
40
|
+
if aggregation.processed?(message_data)
|
41
|
+
logger.info(tag: :ignored) { "Message already handled (Stream: #{message_data.stream_name}, Global Position: #{message_data.global_position})" }
|
42
|
+
return
|
43
|
+
end
|
44
|
+
|
45
|
+
raw_input_data = Messaging::Message::Transform::MessageData.read(message_data)
|
46
|
+
input_metadata = Messaging::Message::Metadata.build(raw_input_data[:metadata])
|
47
|
+
|
48
|
+
output_metadata = Messaging::Message::Metadata.build
|
49
|
+
|
50
|
+
output_metadata.follow(input_metadata)
|
51
|
+
|
52
|
+
output_metadata = output_metadata.to_h
|
53
|
+
output_metadata.delete_if { |_, v| v.nil? }
|
54
|
+
|
55
|
+
write_message_data = MessageStore::MessageData::Write.new
|
56
|
+
|
57
|
+
SetAttributes.(write_message_data, message_data, copy: [:type, :data])
|
58
|
+
|
59
|
+
write_message_data.metadata = output_metadata
|
60
|
+
|
61
|
+
input_category = Messaging::StreamName.get_category(message_data.stream_name)
|
62
|
+
write_message_data = transform(write_message_data, input_category)
|
63
|
+
|
64
|
+
if write_message_data
|
65
|
+
assure_message_data(write_message_data)
|
66
|
+
else
|
67
|
+
logger.info(tag: :ignored) { "Message ignored (Stream: #{message_data.stream_name}, Global Position: #{message_data.global_position})" }
|
68
|
+
return
|
69
|
+
end
|
70
|
+
|
71
|
+
logger.info do
|
72
|
+
message_type = message_data.type
|
73
|
+
unless write_message_data.type == message_type
|
74
|
+
message_type = "#{write_message_data.type} ← #{message_type}"
|
75
|
+
end
|
76
|
+
|
77
|
+
"Message copied (Message Type: #{message_type}, Stream: #{message_data.stream_name}, Global Position: #{message_data.global_position})"
|
78
|
+
end
|
79
|
+
|
80
|
+
Try.(MessageStore::ExpectedVersion::Error) do
|
81
|
+
stream_name = stream_name(stream_id)
|
82
|
+
write.(write_message_data, stream_name, expected_version: version)
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
def transform(write_message_data, stream_name)
|
87
|
+
transform_action.(write_message_data, stream_name)
|
88
|
+
end
|
89
|
+
|
90
|
+
def assure_message_data(message_data)
|
91
|
+
unless message_data.instance_of?(MessageStore::MessageData::Write)
|
92
|
+
raise TransformError, "Not an instance of MessageData::Write"
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
module Configure
|
97
|
+
def configure(session: nil)
|
98
|
+
writer_session = self.writer_session
|
99
|
+
writer_session ||= session
|
100
|
+
|
101
|
+
self.class::Store.configure(self, session: writer_session)
|
102
|
+
|
103
|
+
MessageStore::Postgres::Write.configure(self, session: writer_session)
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
module StoreClass
|
108
|
+
def store_class
|
109
|
+
@store_class ||= Class.new do
|
110
|
+
include Store
|
111
|
+
end
|
112
|
+
end
|
113
|
+
alias_method :store_cls, :store_class
|
114
|
+
end
|
115
|
+
|
116
|
+
module CategoryMacro
|
117
|
+
def category_macro(category)
|
118
|
+
super(category)
|
119
|
+
|
120
|
+
store_class.category_macro(category)
|
121
|
+
end
|
122
|
+
alias_method :category, :category_macro
|
123
|
+
end
|
124
|
+
|
125
|
+
module TransformMacro
|
126
|
+
def transform_macro(&transform_action)
|
127
|
+
define_method(:transform_action) do
|
128
|
+
transform_action
|
129
|
+
end
|
130
|
+
end
|
131
|
+
alias_method :transform, :transform_macro
|
132
|
+
end
|
133
|
+
|
134
|
+
module SnapshotIntervalMacro
|
135
|
+
def snapshot_interval_macro(interval)
|
136
|
+
store_class.snapshot(EntitySnapshot::Postgres, interval: interval)
|
137
|
+
end
|
138
|
+
alias_method :snapshot_interval, :snapshot_interval_macro
|
139
|
+
end
|
140
|
+
|
141
|
+
module WriterSessionMacro
|
142
|
+
def writer_session_macro(&block)
|
143
|
+
define_method(:writer_session, &block)
|
144
|
+
end
|
145
|
+
alias_method :writer_session, :writer_session_macro
|
146
|
+
end
|
147
|
+
end
|
148
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
module AggregateStreams
|
2
|
+
class PositionStore
|
3
|
+
include ::Consumer::PositionStore
|
4
|
+
include ::Log::Dependency
|
5
|
+
include Initializer
|
6
|
+
|
7
|
+
dependency :session, MessageStore::Postgres::Session
|
8
|
+
|
9
|
+
initializer :input_category, :output_category
|
10
|
+
|
11
|
+
def self.build(input_category, output_category, session: nil)
|
12
|
+
instance = new(input_category, output_category)
|
13
|
+
MessageStore::Postgres::Session.configure(instance, session: session)
|
14
|
+
instance
|
15
|
+
end
|
16
|
+
|
17
|
+
def get
|
18
|
+
logger.trace { "Get position (Output Category: #{output_category.inspect}, Input Category: #{input_category.inspect})" }
|
19
|
+
|
20
|
+
sql_command = self.class.get_sql_command
|
21
|
+
|
22
|
+
parameter_values = [output_category, input_category]
|
23
|
+
|
24
|
+
result = session.execute(sql_command, parameter_values)
|
25
|
+
|
26
|
+
if result.ntuples.zero?
|
27
|
+
position = nil
|
28
|
+
else
|
29
|
+
record = result[0]
|
30
|
+
|
31
|
+
position = record['position'].to_i + 1
|
32
|
+
end
|
33
|
+
|
34
|
+
logger.info { "Get position done (Position: #{position || '(none)'}, Output Category: #{output_category.inspect}, Input Category: #{input_category.inspect})" }
|
35
|
+
|
36
|
+
position
|
37
|
+
end
|
38
|
+
|
39
|
+
def self.get_sql_command
|
40
|
+
%{
|
41
|
+
SELECT
|
42
|
+
metadata->>'causationMessageGlobalPosition' AS position
|
43
|
+
FROM messages
|
44
|
+
WHERE
|
45
|
+
category(stream_name) = $1 AND
|
46
|
+
category(metadata->>'causationMessageStreamName') = $2
|
47
|
+
ORDER BY global_position DESC LIMIT 1
|
48
|
+
}
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
module AggregateStreams
|
2
|
+
module Store
|
3
|
+
def self.included(cls)
|
4
|
+
cls.class_exec do
|
5
|
+
include EntityStore
|
6
|
+
|
7
|
+
entity Aggregation
|
8
|
+
projection Projection
|
9
|
+
reader MessageStore::Postgres::Read, batch_size: Defaults.batch_size
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
module Defaults
|
14
|
+
def self.batch_size
|
15
|
+
MessageStore::Postgres::Read::Defaults.batch_size
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
metadata
ADDED
@@ -0,0 +1,143 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: aggregate_streams
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.0.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Nathan Ladd
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2020-11-09 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: evt-consumer-postgres
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: evt-entity_store
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: evt-entity_snapshot-postgres
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: evt-try
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: test_bench
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: evt-component_host
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
97
|
+
description: " "
|
98
|
+
email: nathanladd@gmail.com
|
99
|
+
executables: []
|
100
|
+
extensions: []
|
101
|
+
extra_rdoc_files: []
|
102
|
+
files:
|
103
|
+
- lib/aggregate_streams.rb
|
104
|
+
- lib/aggregate_streams/aggregate_streams.rb
|
105
|
+
- lib/aggregate_streams/aggregation.rb
|
106
|
+
- lib/aggregate_streams/consumer.rb
|
107
|
+
- lib/aggregate_streams/controls.rb
|
108
|
+
- lib/aggregate_streams/controls/aggregation.rb
|
109
|
+
- lib/aggregate_streams/controls/category.rb
|
110
|
+
- lib/aggregate_streams/controls/handler.rb
|
111
|
+
- lib/aggregate_streams/controls/message_data.rb
|
112
|
+
- lib/aggregate_streams/controls/message_data/metadata.rb
|
113
|
+
- lib/aggregate_streams/controls/position.rb
|
114
|
+
- lib/aggregate_streams/controls/store.rb
|
115
|
+
- lib/aggregate_streams/controls/stream_name.rb
|
116
|
+
- lib/aggregate_streams/handle.rb
|
117
|
+
- lib/aggregate_streams/position_store.rb
|
118
|
+
- lib/aggregate_streams/projection.rb
|
119
|
+
- lib/aggregate_streams/store.rb
|
120
|
+
homepage: https://github.com/ntl/aggregate-streams
|
121
|
+
licenses:
|
122
|
+
- MIT
|
123
|
+
metadata: {}
|
124
|
+
post_install_message:
|
125
|
+
rdoc_options: []
|
126
|
+
require_paths:
|
127
|
+
- lib
|
128
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
129
|
+
requirements:
|
130
|
+
- - ">="
|
131
|
+
- !ruby/object:Gem::Version
|
132
|
+
version: '2.7'
|
133
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
134
|
+
requirements:
|
135
|
+
- - ">="
|
136
|
+
- !ruby/object:Gem::Version
|
137
|
+
version: '0'
|
138
|
+
requirements: []
|
139
|
+
rubygems_version: 3.1.4
|
140
|
+
signing_key:
|
141
|
+
specification_version: 4
|
142
|
+
summary: Combine messages from multiple Eventide streams into an aggregate stream
|
143
|
+
test_files: []
|