evt-message_store-postgres 2.4.0.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/lib/message_store/postgres.rb +24 -0
- data/lib/message_store/postgres/controls.rb +7 -0
- data/lib/message_store/postgres/controls/category.rb +7 -0
- data/lib/message_store/postgres/controls/message_data.rb +31 -0
- data/lib/message_store/postgres/controls/position.rb +15 -0
- data/lib/message_store/postgres/controls/put.rb +28 -0
- data/lib/message_store/postgres/controls/stream_name.rb +7 -0
- data/lib/message_store/postgres/get.rb +165 -0
- data/lib/message_store/postgres/get/category.rb +81 -0
- data/lib/message_store/postgres/get/category/consumer_group.rb +20 -0
- data/lib/message_store/postgres/get/category/correlation.rb +17 -0
- data/lib/message_store/postgres/get/condition.rb +15 -0
- data/lib/message_store/postgres/get/stream.rb +67 -0
- data/lib/message_store/postgres/get/stream/last.rb +66 -0
- data/lib/message_store/postgres/log.rb +9 -0
- data/lib/message_store/postgres/put.rb +146 -0
- data/lib/message_store/postgres/read.rb +17 -0
- data/lib/message_store/postgres/session.rb +148 -0
- data/lib/message_store/postgres/settings.rb +36 -0
- data/lib/message_store/postgres/write.rb +52 -0
- metadata +174 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 7898e559251e6013c71e09ad745cacecca46680f54f279d06eb7e4801d5f4fdc
|
4
|
+
data.tar.gz: 251012903cf9814e62eac3db90eee0f9d47467759017d53f81238dac33b0ba32
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: abb2182486ece01a11d7b15866275f4f7872419cb7e966cf9b5e850f717286efb10342466d3ce873072f7f622c0017a2a7bcba41d89495562fad336885fb0156
|
7
|
+
data.tar.gz: 4e7dd8ca3ff0fb4d924bf3fd552f7e00ae14b87d06703ea1a44cdd2f2e98a490d9439f5fa0274cca0a6995e1c79e7be896a57d9c4063b11cffa9b512b2550a46
|
@@ -0,0 +1,24 @@
|
|
1
|
+
require 'pg'
|
2
|
+
|
3
|
+
require 'message_store'
|
4
|
+
|
5
|
+
require 'log'
|
6
|
+
require 'telemetry'
|
7
|
+
require 'settings'
|
8
|
+
|
9
|
+
require 'message_store/postgres/log'
|
10
|
+
|
11
|
+
require 'message_store/postgres/settings'
|
12
|
+
require 'message_store/postgres/session'
|
13
|
+
|
14
|
+
require 'message_store/postgres/put'
|
15
|
+
require 'message_store/postgres/write'
|
16
|
+
|
17
|
+
require 'message_store/postgres/get'
|
18
|
+
require 'message_store/postgres/get/condition'
|
19
|
+
require 'message_store/postgres/get/stream'
|
20
|
+
require 'message_store/postgres/get/stream/last'
|
21
|
+
require 'message_store/postgres/get/category'
|
22
|
+
require 'message_store/postgres/get/category/correlation'
|
23
|
+
require 'message_store/postgres/get/category/consumer_group'
|
24
|
+
require 'message_store/postgres/read'
|
@@ -0,0 +1,7 @@
|
|
1
|
+
require 'message_store/controls'
|
2
|
+
|
3
|
+
require 'message_store/postgres/controls/position'
|
4
|
+
require 'message_store/postgres/controls/category'
|
5
|
+
require 'message_store/postgres/controls/stream_name'
|
6
|
+
require 'message_store/postgres/controls/message_data'
|
7
|
+
require 'message_store/postgres/controls/put'
|
@@ -0,0 +1,31 @@
|
|
1
|
+
module MessageStore
|
2
|
+
module Postgres
|
3
|
+
module Controls
|
4
|
+
MessageData = MessageStore::Controls::MessageData
|
5
|
+
|
6
|
+
module MessageData
|
7
|
+
module Write
|
8
|
+
module List
|
9
|
+
Entry = Struct.new(:stream_name, :category, :message_data)
|
10
|
+
|
11
|
+
def self.get(instances: nil, stream_name: nil, category: nil)
|
12
|
+
instances ||= 1
|
13
|
+
|
14
|
+
list = []
|
15
|
+
instances.times do
|
16
|
+
instance_stream_name = stream_name || StreamName.example(category: category)
|
17
|
+
instance_category = MessageStore::StreamName.get_category(instance_stream_name)
|
18
|
+
|
19
|
+
write_message = Controls::MessageData::Write.example
|
20
|
+
|
21
|
+
list << Entry.new(instance_stream_name, instance_category, write_message)
|
22
|
+
end
|
23
|
+
|
24
|
+
list
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module MessageStore
|
2
|
+
module Postgres
|
3
|
+
module Controls
|
4
|
+
module Put
|
5
|
+
def self.call(instances: nil, stream_name: nil, message_data: nil, message: nil, category: nil)
|
6
|
+
instances ||= 1
|
7
|
+
stream_name ||= StreamName.example(category: category)
|
8
|
+
message_data ||= message
|
9
|
+
|
10
|
+
message_specified = !message_data.nil?
|
11
|
+
|
12
|
+
message_data ||= MessageData::Write.example
|
13
|
+
|
14
|
+
position = nil
|
15
|
+
instances.times do
|
16
|
+
position = MessageStore::Postgres::Put.(message_data, stream_name)
|
17
|
+
|
18
|
+
unless message_specified
|
19
|
+
message_data.id = MessageData::Write.id
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
[stream_name, position]
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,165 @@
|
|
1
|
+
module MessageStore
|
2
|
+
module Postgres
|
3
|
+
module Get
|
4
|
+
def self.included(cls)
|
5
|
+
cls.class_exec do
|
6
|
+
include MessageStore::Get
|
7
|
+
|
8
|
+
prepend Call
|
9
|
+
prepend BatchSize
|
10
|
+
|
11
|
+
dependency :session, Session
|
12
|
+
|
13
|
+
abstract :stream_name
|
14
|
+
abstract :sql_command
|
15
|
+
abstract :parameters
|
16
|
+
abstract :parameter_values
|
17
|
+
abstract :last_position
|
18
|
+
abstract :log_text
|
19
|
+
|
20
|
+
virtual :specialize_error
|
21
|
+
virtual :assure
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
module BatchSize
|
26
|
+
def batch_size
|
27
|
+
@batch_size ||= Defaults.batch_size
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def self.build(stream_name, **args)
|
32
|
+
cls = specialization(stream_name)
|
33
|
+
cls.build(stream_name, **args)
|
34
|
+
end
|
35
|
+
|
36
|
+
def self.configure(receiver, stream_name, **args)
|
37
|
+
attr_name = args.delete(:attr_name)
|
38
|
+
attr_name ||= :get
|
39
|
+
|
40
|
+
instance = build(stream_name, **args)
|
41
|
+
receiver.public_send("#{attr_name}=", instance)
|
42
|
+
end
|
43
|
+
|
44
|
+
def configure(session: nil)
|
45
|
+
Session.configure(self, session: session)
|
46
|
+
end
|
47
|
+
|
48
|
+
def self.call(stream_name, **args)
|
49
|
+
position = args.delete(:position)
|
50
|
+
instance = build(stream_name, **args)
|
51
|
+
instance.(position)
|
52
|
+
end
|
53
|
+
|
54
|
+
module Call
|
55
|
+
def call(position=nil, stream_name: nil)
|
56
|
+
position ||= self.class::Defaults.position
|
57
|
+
|
58
|
+
stream_name ||= self.stream_name
|
59
|
+
|
60
|
+
assure
|
61
|
+
|
62
|
+
logger.trace(tag: :get) { "Getting message data (#{log_text(stream_name, position)})" }
|
63
|
+
|
64
|
+
result = get_result(stream_name, position)
|
65
|
+
|
66
|
+
message_data = convert(result)
|
67
|
+
|
68
|
+
logger.info(tag: :get) { "Finished getting message data (Count: #{message_data.length}, #{log_text(stream_name, position)})" }
|
69
|
+
logger.info(tags: [:data, :message_data]) { message_data.pretty_inspect }
|
70
|
+
|
71
|
+
message_data
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
def get_result(stream_name, position)
|
76
|
+
logger.trace(tag: :get) { "Getting result (#{log_text(stream_name, position)})" }
|
77
|
+
|
78
|
+
parameter_values = parameter_values(stream_name, position)
|
79
|
+
|
80
|
+
begin
|
81
|
+
result = session.execute(sql_command, parameter_values)
|
82
|
+
rescue PG::RaiseException => e
|
83
|
+
raise_error(e)
|
84
|
+
end
|
85
|
+
|
86
|
+
logger.debug(tag: :get) { "Finished getting result (Count: #{result.ntuples}, #{log_text(stream_name, position)})" }
|
87
|
+
|
88
|
+
result
|
89
|
+
end
|
90
|
+
|
91
|
+
def convert(result)
|
92
|
+
logger.trace(tag: :get) { "Converting result to message data (Result Count: #{result.ntuples})" }
|
93
|
+
|
94
|
+
message_data = result.map do |record|
|
95
|
+
Get.message_data(record)
|
96
|
+
end
|
97
|
+
|
98
|
+
logger.debug(tag: :get) { "Converted result to message data (Message Data Count: #{message_data.length})" }
|
99
|
+
|
100
|
+
message_data
|
101
|
+
end
|
102
|
+
|
103
|
+
def self.message_data(record)
|
104
|
+
record['data'] = Get::Deserialize.data(record['data'])
|
105
|
+
record['metadata'] = Get::Deserialize.metadata(record['metadata'])
|
106
|
+
record['time'] = Get::Time.utc_coerced(record['time'])
|
107
|
+
|
108
|
+
MessageData::Read.build(record)
|
109
|
+
end
|
110
|
+
|
111
|
+
def raise_error(pg_error)
|
112
|
+
error_message = Get.error_message(pg_error)
|
113
|
+
|
114
|
+
error = Condition.error(error_message)
|
115
|
+
|
116
|
+
if error.nil?
|
117
|
+
error = specialize_error(error_message)
|
118
|
+
end
|
119
|
+
|
120
|
+
if not error.nil?
|
121
|
+
logger.error { error_message }
|
122
|
+
raise error
|
123
|
+
end
|
124
|
+
|
125
|
+
raise pg_error
|
126
|
+
end
|
127
|
+
|
128
|
+
def self.error_message(pg_error)
|
129
|
+
pg_error.message.gsub('ERROR:', '').strip
|
130
|
+
end
|
131
|
+
|
132
|
+
def self.specialization(stream_name)
|
133
|
+
if StreamName.category?(stream_name)
|
134
|
+
Category
|
135
|
+
else
|
136
|
+
Stream
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
140
|
+
module Deserialize
|
141
|
+
def self.data(serialized_data)
|
142
|
+
return nil if serialized_data.nil?
|
143
|
+
Transform::Read.(serialized_data, :json, MessageData::Hash)
|
144
|
+
end
|
145
|
+
|
146
|
+
def self.metadata(serialized_metadata)
|
147
|
+
return nil if serialized_metadata.nil?
|
148
|
+
Transform::Read.(serialized_metadata, :json, MessageData::Hash)
|
149
|
+
end
|
150
|
+
end
|
151
|
+
|
152
|
+
module Time
|
153
|
+
def self.utc_coerced(local_time)
|
154
|
+
Clock::UTC.coerce(local_time)
|
155
|
+
end
|
156
|
+
end
|
157
|
+
|
158
|
+
module Defaults
|
159
|
+
def self.batch_size
|
160
|
+
1000
|
161
|
+
end
|
162
|
+
end
|
163
|
+
end
|
164
|
+
end
|
165
|
+
end
|
@@ -0,0 +1,81 @@
|
|
1
|
+
module MessageStore
|
2
|
+
module Postgres
|
3
|
+
module Get
|
4
|
+
class Category
|
5
|
+
Error = Class.new(RuntimeError)
|
6
|
+
|
7
|
+
include Get
|
8
|
+
|
9
|
+
initializer :category, na(:batch_size), :correlation, :consumer_group_member, :consumer_group_size, :condition
|
10
|
+
alias :stream_name :category
|
11
|
+
|
12
|
+
def self.call(category, position: nil, batch_size: nil, correlation: nil, consumer_group_member: nil, consumer_group_size: nil, condition: nil, session: nil)
|
13
|
+
instance = build(category, batch_size: batch_size, correlation: correlation, consumer_group_member: consumer_group_member, consumer_group_size: consumer_group_size, condition: condition, session: session)
|
14
|
+
instance.(position)
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.build(category, batch_size: nil, correlation: nil, consumer_group_member: nil, consumer_group_size: nil, condition: nil, session: nil)
|
18
|
+
instance = new(category, batch_size, correlation, consumer_group_member, consumer_group_size, condition)
|
19
|
+
instance.configure(session: session)
|
20
|
+
instance
|
21
|
+
end
|
22
|
+
|
23
|
+
def self.configure(receiver, category, attr_name: nil, batch_size: nil, correlation: nil, consumer_group_member: nil, consumer_group_size: nil, condition: nil, session: nil)
|
24
|
+
attr_name ||= :get
|
25
|
+
instance = build(category, batch_size: batch_size, correlation: correlation, consumer_group_member: consumer_group_member, consumer_group_size: consumer_group_size, condition: condition, session: session)
|
26
|
+
receiver.public_send("#{attr_name}=", instance)
|
27
|
+
end
|
28
|
+
|
29
|
+
def sql_command
|
30
|
+
"SELECT * FROM get_category_messages(#{parameters});"
|
31
|
+
end
|
32
|
+
|
33
|
+
def parameters
|
34
|
+
'$1::varchar, $2::bigint, $3::bigint, $4::varchar, $5::bigint, $6::bigint, $7::varchar'
|
35
|
+
end
|
36
|
+
|
37
|
+
def parameter_values(category, position)
|
38
|
+
[
|
39
|
+
category,
|
40
|
+
position,
|
41
|
+
batch_size,
|
42
|
+
correlation,
|
43
|
+
consumer_group_member,
|
44
|
+
consumer_group_size,
|
45
|
+
condition
|
46
|
+
]
|
47
|
+
end
|
48
|
+
|
49
|
+
def last_position(batch)
|
50
|
+
batch.last.global_position
|
51
|
+
end
|
52
|
+
|
53
|
+
def specialize_error(error_message)
|
54
|
+
error = Correlation.error(error_message)
|
55
|
+
|
56
|
+
if error.nil?
|
57
|
+
error = ConsumerGroup.error(error_message)
|
58
|
+
end
|
59
|
+
|
60
|
+
error
|
61
|
+
end
|
62
|
+
|
63
|
+
def log_text(category, position)
|
64
|
+
"Category: #{category}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Correlation: #{correlation.inspect}, Consumer Group Member: #{consumer_group_member.inspect}, Consumer Group Size: #{consumer_group_size.inspect}, Condition: #{condition.inspect})"
|
65
|
+
end
|
66
|
+
|
67
|
+
def assure
|
68
|
+
if not MessageStore::StreamName.category?(category)
|
69
|
+
raise Error, "Must be a category (Stream Name: #{category})"
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
module Defaults
|
74
|
+
def self.position
|
75
|
+
1
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
module MessageStore
|
2
|
+
module Postgres
|
3
|
+
module Get
|
4
|
+
class Category
|
5
|
+
module ConsumerGroup
|
6
|
+
Error = Class.new(RuntimeError)
|
7
|
+
|
8
|
+
def self.error(error_message)
|
9
|
+
if error_message.start_with?('Consumer group size must not be less than 1') ||
|
10
|
+
error_message.start_with?('Consumer group member must be less than the group size') ||
|
11
|
+
error_message.start_with?('Consumer group member must not be less than 0') ||
|
12
|
+
error_message.start_with?('Consumer group member and size must be specified')
|
13
|
+
Error.new(error_message)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
module MessageStore
|
2
|
+
module Postgres
|
3
|
+
module Get
|
4
|
+
class Category
|
5
|
+
module Correlation
|
6
|
+
Error = Class.new(RuntimeError)
|
7
|
+
|
8
|
+
def self.error(error_message)
|
9
|
+
if error_message.start_with?('Correlation must be a category')
|
10
|
+
Error.new(error_message)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
module MessageStore
|
2
|
+
module Postgres
|
3
|
+
module Get
|
4
|
+
module Condition
|
5
|
+
Error = Class.new(RuntimeError)
|
6
|
+
|
7
|
+
def self.error(error_message)
|
8
|
+
if error_message.start_with?('Retrieval with SQL condition is not activated')
|
9
|
+
Get::Condition::Error.new(error_message)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,67 @@
|
|
1
|
+
module MessageStore
|
2
|
+
module Postgres
|
3
|
+
module Get
|
4
|
+
class Stream
|
5
|
+
Error = Class.new(RuntimeError)
|
6
|
+
|
7
|
+
include Get
|
8
|
+
|
9
|
+
initializer :stream_name, na(:batch_size), :condition
|
10
|
+
|
11
|
+
def self.call(stream_name, position: nil, batch_size: nil, condition: nil, session: nil)
|
12
|
+
instance = build(stream_name, batch_size: batch_size, condition: condition, session: session)
|
13
|
+
instance.(position)
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.build(stream_name, batch_size: nil, condition: nil, session: nil)
|
17
|
+
instance = new(stream_name, batch_size, condition)
|
18
|
+
instance.configure(session: session)
|
19
|
+
instance
|
20
|
+
end
|
21
|
+
|
22
|
+
def self.configure(receiver, stream_name, attr_name: nil, batch_size: nil, condition: nil, session: nil)
|
23
|
+
attr_name ||= :get
|
24
|
+
instance = build(stream_name, batch_size: batch_size, condition: condition, session: session)
|
25
|
+
receiver.public_send("#{attr_name}=", instance)
|
26
|
+
end
|
27
|
+
|
28
|
+
def sql_command
|
29
|
+
"SELECT * FROM get_stream_messages(#{parameters});"
|
30
|
+
end
|
31
|
+
|
32
|
+
def parameters
|
33
|
+
'$1::varchar, $2::bigint, $3::bigint, $4::varchar'
|
34
|
+
end
|
35
|
+
|
36
|
+
def parameter_values(stream_name, position)
|
37
|
+
[
|
38
|
+
stream_name,
|
39
|
+
position,
|
40
|
+
batch_size,
|
41
|
+
condition
|
42
|
+
]
|
43
|
+
end
|
44
|
+
|
45
|
+
def last_position(batch)
|
46
|
+
batch.last.position
|
47
|
+
end
|
48
|
+
|
49
|
+
def log_text(stream_name, position)
|
50
|
+
"Stream Name: #{stream_name}, Position: #{position.inspect}, Batch Size: #{batch_size.inspect}, Condition: #{condition.inspect})"
|
51
|
+
end
|
52
|
+
|
53
|
+
def assure
|
54
|
+
if MessageStore::StreamName.category?(stream_name)
|
55
|
+
raise Error, "Must be a stream name (Category: #{stream_name})"
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
module Defaults
|
60
|
+
def self.position
|
61
|
+
0
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
@@ -0,0 +1,66 @@
|
|
1
|
+
module MessageStore
|
2
|
+
module Postgres
|
3
|
+
module Get
|
4
|
+
class Stream
|
5
|
+
class Last
|
6
|
+
include MessageStore::Get::Stream::Last
|
7
|
+
|
8
|
+
dependency :session, Session
|
9
|
+
|
10
|
+
def configure(session: nil)
|
11
|
+
Session.configure(self, session: session)
|
12
|
+
end
|
13
|
+
|
14
|
+
def call(stream_name)
|
15
|
+
logger.trace(tag: :get) { "Getting last message data (Stream Name: #{stream_name})" }
|
16
|
+
|
17
|
+
result = get_result(stream_name)
|
18
|
+
|
19
|
+
return nil if result.nil?
|
20
|
+
|
21
|
+
message_data = convert(result[0])
|
22
|
+
|
23
|
+
logger.info(tag: :get) { "Finished getting message data (Stream Name: #{stream_name})" }
|
24
|
+
logger.info(tags: [:data, :message_data]) { message_data.pretty_inspect }
|
25
|
+
|
26
|
+
message_data
|
27
|
+
end
|
28
|
+
|
29
|
+
def get_result(stream_name)
|
30
|
+
logger.trace(tag: :get) { "Getting last record (Stream: #{stream_name})" }
|
31
|
+
|
32
|
+
sql_command = self.class.sql_command(stream_name)
|
33
|
+
|
34
|
+
parameter_values = [
|
35
|
+
stream_name
|
36
|
+
]
|
37
|
+
|
38
|
+
result = session.execute(sql_command, parameter_values)
|
39
|
+
|
40
|
+
logger.debug(tag: :get) { "Finished getting result (Count: #{result.ntuples}, Stream: #{stream_name}" }
|
41
|
+
|
42
|
+
return nil if result.ntuples == 0
|
43
|
+
|
44
|
+
result
|
45
|
+
end
|
46
|
+
|
47
|
+
def self.sql_command(stream_name)
|
48
|
+
parameters = '$1::varchar'
|
49
|
+
|
50
|
+
"SELECT * FROM get_last_stream_message(#{parameters});"
|
51
|
+
end
|
52
|
+
|
53
|
+
def convert(record)
|
54
|
+
logger.trace(tag: :get) { "Converting record to message data" }
|
55
|
+
|
56
|
+
message_data = Get.message_data(record)
|
57
|
+
|
58
|
+
logger.debug(tag: :get) { "Converted record to message data" }
|
59
|
+
|
60
|
+
message_data
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
@@ -0,0 +1,146 @@
|
|
1
|
+
module MessageStore
|
2
|
+
module Postgres
|
3
|
+
class Put
|
4
|
+
include Dependency
|
5
|
+
include Log::Dependency
|
6
|
+
|
7
|
+
dependency :session, Session
|
8
|
+
dependency :identifier, Identifier::UUID::Random
|
9
|
+
|
10
|
+
def self.build(session: nil)
|
11
|
+
new.tap do |instance|
|
12
|
+
instance.configure(session: session)
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def configure(session: nil)
|
17
|
+
Session.configure(self, session: session)
|
18
|
+
Identifier::UUID::Random.configure(self)
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.configure(receiver, session: nil, attr_name: nil)
|
22
|
+
attr_name ||= :put
|
23
|
+
instance = build(session: session)
|
24
|
+
receiver.public_send "#{attr_name}=", instance
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.call(write_message, stream_name, expected_version: nil, session: nil)
|
28
|
+
instance = build(session: session)
|
29
|
+
instance.(write_message, stream_name, expected_version: expected_version)
|
30
|
+
end
|
31
|
+
|
32
|
+
def call(write_message, stream_name, expected_version: nil)
|
33
|
+
logger.trace(tag: :put) { "Putting message data (Type: #{write_message.type}, Stream Name: #{stream_name}, Expected Version: #{expected_version.inspect})" }
|
34
|
+
logger.trace(tags: [:data, :message_data]) { write_message.pretty_inspect }
|
35
|
+
|
36
|
+
write_message.id ||= identifier.get
|
37
|
+
|
38
|
+
id, type, data, metadata = destructure_message(write_message)
|
39
|
+
expected_version = ExpectedVersion.canonize(expected_version)
|
40
|
+
|
41
|
+
insert_message(id, stream_name, type, data, metadata, expected_version).tap do |position|
|
42
|
+
logger.info(tag: :put) { "Put message data (Type: #{write_message.type}, Stream Name: #{stream_name}, Expected Version: #{expected_version.inspect}, ID: #{id.inspect}, Position: #{position})" }
|
43
|
+
logger.info(tags: [:data, :message_data]) { write_message.pretty_inspect }
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def destructure_message(write_message)
|
48
|
+
id = write_message.id
|
49
|
+
type = write_message.type
|
50
|
+
data = write_message.data
|
51
|
+
metadata = write_message.metadata
|
52
|
+
|
53
|
+
logger.debug(tags: [:data, :message_data]) { "ID: #{id.pretty_inspect}" }
|
54
|
+
logger.debug(tags: [:data, :message_data]) { "Type: #{type.pretty_inspect}" }
|
55
|
+
logger.debug(tags: [:data, :message_data]) { "Data: #{data.pretty_inspect}" }
|
56
|
+
logger.debug(tags: [:data, :message_data]) { "Metadata: #{metadata.pretty_inspect}" }
|
57
|
+
|
58
|
+
return id, type, data, metadata
|
59
|
+
end
|
60
|
+
|
61
|
+
def insert_message(id, stream_name, type, data, metadata, expected_version)
|
62
|
+
transformed_data = transformed_data(data)
|
63
|
+
transformed_metadata = transformed_metadata(metadata)
|
64
|
+
records = execute_query(id, stream_name, type, transformed_data, transformed_metadata, expected_version)
|
65
|
+
position(records)
|
66
|
+
end
|
67
|
+
|
68
|
+
def execute_query(id, stream_name, type, transformed_data, transformed_metadata, expected_version)
|
69
|
+
logger.trace(tag: :put) { "Executing insert (Stream Name: #{stream_name}, Type: #{type}, Expected Version: #{expected_version.inspect}, ID: #{id.inspect})" }
|
70
|
+
|
71
|
+
params = [
|
72
|
+
id,
|
73
|
+
stream_name,
|
74
|
+
type,
|
75
|
+
transformed_data,
|
76
|
+
transformed_metadata,
|
77
|
+
expected_version
|
78
|
+
]
|
79
|
+
|
80
|
+
begin
|
81
|
+
records = session.execute(self.class.statement, params)
|
82
|
+
rescue PG::RaiseException => e
|
83
|
+
raise_error e
|
84
|
+
end
|
85
|
+
|
86
|
+
logger.debug(tag: :put) { "Executed insert (Type: #{type}, Stream Name: #{stream_name}, Expected Version: #{expected_version.inspect}, ID: #{id.inspect})" }
|
87
|
+
|
88
|
+
records
|
89
|
+
end
|
90
|
+
|
91
|
+
def self.statement
|
92
|
+
@statement ||= "SELECT write_message($1::varchar, $2::varchar, $3::varchar, $4::jsonb, $5::jsonb, $6::bigint);"
|
93
|
+
end
|
94
|
+
|
95
|
+
def transformed_data(data)
|
96
|
+
transformed_data = nil
|
97
|
+
|
98
|
+
if data.is_a?(Hash) && data.empty?
|
99
|
+
data = nil
|
100
|
+
end
|
101
|
+
|
102
|
+
unless data.nil?
|
103
|
+
transformable_data = MessageData::Hash[data]
|
104
|
+
transformed_data = Transform::Write.(transformable_data, :json)
|
105
|
+
end
|
106
|
+
|
107
|
+
logger.debug(tags: [:data, :serialize]) { "Transformed Data: #{transformed_data.inspect}" }
|
108
|
+
transformed_data
|
109
|
+
end
|
110
|
+
|
111
|
+
def transformed_metadata(metadata)
|
112
|
+
transformed_metadata = nil
|
113
|
+
|
114
|
+
if metadata.is_a?(Hash) && metadata.empty?
|
115
|
+
metadata = nil
|
116
|
+
end
|
117
|
+
|
118
|
+
unless metadata.nil?
|
119
|
+
transformable_metadata = MessageData::Hash[metadata]
|
120
|
+
transformed_metadata = Transform::Write.(transformable_metadata, :json)
|
121
|
+
end
|
122
|
+
|
123
|
+
logger.debug(tags: [:data, :serialize]) { "Transformed Metadata: #{transformed_metadata.inspect}" }
|
124
|
+
transformed_metadata
|
125
|
+
end
|
126
|
+
|
127
|
+
def position(records)
|
128
|
+
position = nil
|
129
|
+
unless records[0].nil?
|
130
|
+
position = records[0].values[0]
|
131
|
+
end
|
132
|
+
position
|
133
|
+
end
|
134
|
+
|
135
|
+
def raise_error(pg_error)
|
136
|
+
error_message = pg_error.message
|
137
|
+
if error_message.include? 'Wrong expected version'
|
138
|
+
error_message.gsub!('ERROR:', '').strip!
|
139
|
+
logger.error { error_message }
|
140
|
+
raise ExpectedVersion::Error, error_message
|
141
|
+
end
|
142
|
+
raise pg_error
|
143
|
+
end
|
144
|
+
end
|
145
|
+
end
|
146
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
module MessageStore
|
2
|
+
module Postgres
|
3
|
+
class Read
|
4
|
+
include MessageStore::Read
|
5
|
+
|
6
|
+
def configure(session: nil, condition: nil)
|
7
|
+
Get.configure(self.iterator, self.stream_name, batch_size: batch_size, condition: condition, session: session)
|
8
|
+
end
|
9
|
+
|
10
|
+
module Defaults
|
11
|
+
def self.batch_size
|
12
|
+
Postgres::Get::Defaults.batch_size
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,148 @@
|
|
1
|
+
module MessageStore
|
2
|
+
module Postgres
|
3
|
+
class Session
|
4
|
+
Error = Class.new(RuntimeError)
|
5
|
+
|
6
|
+
include Settings::Setting
|
7
|
+
|
8
|
+
include Log::Dependency
|
9
|
+
|
10
|
+
def self.settings
|
11
|
+
Settings.names
|
12
|
+
end
|
13
|
+
|
14
|
+
settings.each do |s|
|
15
|
+
setting s
|
16
|
+
end
|
17
|
+
|
18
|
+
attr_accessor :connection
|
19
|
+
|
20
|
+
def self.build(settings: nil)
|
21
|
+
new.tap do |instance|
|
22
|
+
settings ||= Settings.instance
|
23
|
+
settings.set(instance)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.configure(receiver, session: nil, settings: nil, attr_name: nil)
|
28
|
+
attr_name ||= :session
|
29
|
+
|
30
|
+
if session != nil && settings != nil
|
31
|
+
error_msg = "Session configured with both settings and session arguments. Use one or the other, but not both."
|
32
|
+
logger.error(tag: :session) { error_msg }
|
33
|
+
raise Error, error_msg
|
34
|
+
end
|
35
|
+
|
36
|
+
instance = session || build(settings: settings)
|
37
|
+
receiver.public_send "#{attr_name}=", instance
|
38
|
+
instance
|
39
|
+
end
|
40
|
+
|
41
|
+
def open
|
42
|
+
logger.trace(tag: :session) { "Connecting to database" }
|
43
|
+
|
44
|
+
if connected?
|
45
|
+
logger.debug(tag: :session) { "Already connected. A new connection will not be built." }
|
46
|
+
return connection
|
47
|
+
end
|
48
|
+
|
49
|
+
logger.debug(tag: :session) { "Not connected. A new connection will be built." }
|
50
|
+
connection = self.class.build_connection(self)
|
51
|
+
self.connection = connection
|
52
|
+
|
53
|
+
logger.debug(tag: :session) { "Connected to database" }
|
54
|
+
|
55
|
+
connection
|
56
|
+
end
|
57
|
+
alias :connect :open
|
58
|
+
|
59
|
+
def self.build_connection(instance)
|
60
|
+
settings = instance.settings
|
61
|
+
logger.trace(tag: :session) { "Building new connection to database (Settings: #{LogText.settings(settings).inspect})" }
|
62
|
+
|
63
|
+
connection = PG::Connection.open(settings)
|
64
|
+
connection.type_map_for_results = PG::BasicTypeMapForResults.new(connection)
|
65
|
+
|
66
|
+
logger.debug(tag: :session) { "Built new connection to database (Settings: #{LogText.settings(settings).inspect})" }
|
67
|
+
|
68
|
+
connection
|
69
|
+
end
|
70
|
+
|
71
|
+
def connected?
|
72
|
+
return false if connection.nil?
|
73
|
+
|
74
|
+
status = PG::CONNECTION_OK
|
75
|
+
begin
|
76
|
+
status = connection.status
|
77
|
+
rescue PG::ConnectionBad
|
78
|
+
status = nil
|
79
|
+
end
|
80
|
+
|
81
|
+
status == PG::CONNECTION_OK
|
82
|
+
end
|
83
|
+
alias :open? :connected?
|
84
|
+
|
85
|
+
def close
|
86
|
+
connection.close
|
87
|
+
connection = nil
|
88
|
+
end
|
89
|
+
|
90
|
+
def reset
|
91
|
+
connection.reset
|
92
|
+
end
|
93
|
+
|
94
|
+
def settings
|
95
|
+
settings = {}
|
96
|
+
self.class.settings.each do |s|
|
97
|
+
val = public_send(s)
|
98
|
+
settings[s] = val unless val.nil?
|
99
|
+
end
|
100
|
+
settings
|
101
|
+
end
|
102
|
+
|
103
|
+
def execute(sql_command, params=nil)
|
104
|
+
logger.trace(tag: :session) { "Executing SQL command" }
|
105
|
+
logger.trace(tag: :sql) { sql_command }
|
106
|
+
logger.trace(tag: :data) { params.pretty_inspect }
|
107
|
+
|
108
|
+
unless connected?
|
109
|
+
connect
|
110
|
+
end
|
111
|
+
|
112
|
+
if params.nil?
|
113
|
+
connection.exec(sql_command).tap do
|
114
|
+
logger.debug(tag: :session) { "Executed SQL command (no params)" }
|
115
|
+
end
|
116
|
+
else
|
117
|
+
connection.exec_params(sql_command, params).tap do
|
118
|
+
logger.debug(tag: :session) { "Executed SQL command with params" }
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
def transaction(&blk)
|
124
|
+
unless connected?
|
125
|
+
connect
|
126
|
+
end
|
127
|
+
|
128
|
+
connection.transaction(&blk)
|
129
|
+
end
|
130
|
+
|
131
|
+
def self.logger
|
132
|
+
@logger ||= Log.get self
|
133
|
+
end
|
134
|
+
|
135
|
+
module LogText
|
136
|
+
def self.settings(settings)
|
137
|
+
s = settings.dup
|
138
|
+
|
139
|
+
if s.has_key?(:password)
|
140
|
+
s[:password] = '*' * 8
|
141
|
+
end
|
142
|
+
|
143
|
+
s
|
144
|
+
end
|
145
|
+
end
|
146
|
+
end
|
147
|
+
end
|
148
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
module MessageStore
|
2
|
+
module Postgres
|
3
|
+
class Settings < ::Settings
|
4
|
+
def self.instance
|
5
|
+
@instance ||= build
|
6
|
+
end
|
7
|
+
|
8
|
+
def self.data_source
|
9
|
+
Defaults.data_source
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.names
|
13
|
+
[
|
14
|
+
:dbname,
|
15
|
+
:host,
|
16
|
+
:hostaddr,
|
17
|
+
:port,
|
18
|
+
:user,
|
19
|
+
:password,
|
20
|
+
:connect_timeout,
|
21
|
+
:options,
|
22
|
+
:sslmode,
|
23
|
+
:krbsrvname,
|
24
|
+
:gsslib,
|
25
|
+
:service
|
26
|
+
]
|
27
|
+
end
|
28
|
+
|
29
|
+
class Defaults
|
30
|
+
def self.data_source
|
31
|
+
ENV['MESSAGE_STORE_SETTINGS_PATH'] || 'settings/message_store_postgres.json'
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
module MessageStore
|
2
|
+
module Postgres
|
3
|
+
class Write
|
4
|
+
include MessageStore::Write
|
5
|
+
|
6
|
+
dependency :put
|
7
|
+
|
8
|
+
def configure(session: nil)
|
9
|
+
Put.configure(self, session: session)
|
10
|
+
end
|
11
|
+
|
12
|
+
def write(batch, stream_name, expected_version: nil)
|
13
|
+
logger.trace(tag: :write) do
|
14
|
+
message_types = batch.map {|message_data| message_data.type }.uniq.join(', ')
|
15
|
+
"Writing batch (Stream Name: #{stream_name}, Types: #{message_types}, Number of Messages: #{batch.length}, Expected Version: #{expected_version.inspect})"
|
16
|
+
end
|
17
|
+
|
18
|
+
unless expected_version.nil?
|
19
|
+
expected_version = ExpectedVersion.canonize(expected_version)
|
20
|
+
end
|
21
|
+
|
22
|
+
last_position = nil
|
23
|
+
put.session.transaction do
|
24
|
+
batch.each do |message_data|
|
25
|
+
last_position = write_message_data(message_data, stream_name, expected_version: expected_version)
|
26
|
+
|
27
|
+
unless expected_version.nil?
|
28
|
+
expected_version += 1
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
logger.debug(tag: :write) do
|
34
|
+
message_types = batch.map {|message_data| message_data.type }.uniq.join(', ')
|
35
|
+
"Wrote batch (Stream Name: #{stream_name}, Types: #{message_types}, Number of Messages: #{batch.length}, Expected Version: #{expected_version.inspect})"
|
36
|
+
end
|
37
|
+
|
38
|
+
last_position
|
39
|
+
end
|
40
|
+
|
41
|
+
def write_message_data(message_data, stream_name, expected_version: nil)
|
42
|
+
logger.trace(tag: :write) { "Writing message data (Stream Name: #{stream_name}, Type: #{message_data.type}, Expected Version: #{expected_version.inspect})" }
|
43
|
+
logger.trace(tags: [:data, :message_data]) { message_data.pretty_inspect }
|
44
|
+
|
45
|
+
put.(message_data, stream_name, expected_version: expected_version).tap do
|
46
|
+
logger.debug(tag: :write) { "Wrote message data (Stream Name: #{stream_name}, Type: #{message_data.type}, Expected Version: #{expected_version.inspect})" }
|
47
|
+
logger.debug(tags: [:data, :message_data]) { message_data.pretty_inspect }
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
metadata
ADDED
@@ -0,0 +1,174 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: evt-message_store-postgres
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 2.4.0.5
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- The Eventide Project
|
8
|
+
autorequire:
|
9
|
+
bindir: scripts
|
10
|
+
cert_chain: []
|
11
|
+
date: 2020-07-21 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: evt-message_store
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: evt-log
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: evt-settings
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: message-db
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: pg
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :runtime
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: test_bench
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: evt-diagnostics-sample
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: ntl-actor
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ">="
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
125
|
+
description: " "
|
126
|
+
email: opensource@eventide-project.org
|
127
|
+
executables: []
|
128
|
+
extensions: []
|
129
|
+
extra_rdoc_files: []
|
130
|
+
files:
|
131
|
+
- lib/message_store/postgres.rb
|
132
|
+
- lib/message_store/postgres/controls.rb
|
133
|
+
- lib/message_store/postgres/controls/category.rb
|
134
|
+
- lib/message_store/postgres/controls/message_data.rb
|
135
|
+
- lib/message_store/postgres/controls/position.rb
|
136
|
+
- lib/message_store/postgres/controls/put.rb
|
137
|
+
- lib/message_store/postgres/controls/stream_name.rb
|
138
|
+
- lib/message_store/postgres/get.rb
|
139
|
+
- lib/message_store/postgres/get/category.rb
|
140
|
+
- lib/message_store/postgres/get/category/consumer_group.rb
|
141
|
+
- lib/message_store/postgres/get/category/correlation.rb
|
142
|
+
- lib/message_store/postgres/get/condition.rb
|
143
|
+
- lib/message_store/postgres/get/stream.rb
|
144
|
+
- lib/message_store/postgres/get/stream/last.rb
|
145
|
+
- lib/message_store/postgres/log.rb
|
146
|
+
- lib/message_store/postgres/put.rb
|
147
|
+
- lib/message_store/postgres/read.rb
|
148
|
+
- lib/message_store/postgres/session.rb
|
149
|
+
- lib/message_store/postgres/settings.rb
|
150
|
+
- lib/message_store/postgres/write.rb
|
151
|
+
homepage: https://github.com/eventide-project/message-store-postgres
|
152
|
+
licenses:
|
153
|
+
- MIT
|
154
|
+
metadata: {}
|
155
|
+
post_install_message:
|
156
|
+
rdoc_options: []
|
157
|
+
require_paths:
|
158
|
+
- lib
|
159
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
160
|
+
requirements:
|
161
|
+
- - ">="
|
162
|
+
- !ruby/object:Gem::Version
|
163
|
+
version: 2.4.0
|
164
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
165
|
+
requirements:
|
166
|
+
- - ">="
|
167
|
+
- !ruby/object:Gem::Version
|
168
|
+
version: '0'
|
169
|
+
requirements: []
|
170
|
+
rubygems_version: 3.1.2
|
171
|
+
signing_key:
|
172
|
+
specification_version: 4
|
173
|
+
summary: Message store implementation for PostgreSQL
|
174
|
+
test_files: []
|