messaging 3.5.7 → 3.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.circleci/config.yml +8 -8
- data/lib/messaging/adapters/kafka/producer.rb +1 -1
- data/lib/messaging/adapters/postgres/categories/row.rb +30 -0
- data/lib/messaging/adapters/postgres/categories.rb +70 -7
- data/lib/messaging/adapters/postgres/category.rb +7 -21
- data/lib/messaging/adapters/postgres/category_with_partitions.rb +89 -0
- data/lib/messaging/adapters/postgres/serialized_message.rb +1 -1
- data/lib/messaging/adapters/postgres/store.rb +16 -6
- data/lib/messaging/adapters/postgres.rb +32 -0
- data/lib/messaging/adapters/test/category.rb +0 -4
- data/lib/messaging/message.rb +4 -0
- data/lib/messaging/version.rb +1 -1
- data/lib/messaging.rb +11 -0
- metadata +5 -4
- data/.ruby-version +0 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 860c436d2faab463d40144358859eb282aa352370ffb2c725cfa292c267eb959
|
4
|
+
data.tar.gz: cf39e2bf8a10ebe84cfe09d9e74d0fa9bce654803478865b47601b5ada04d404
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: '0751658ec0a77cc8fd925963c1530c05ee31cb458c8f773a8aba7dd333569e90547945d52dda5cbee21e0f0668cf6d7909cb2f2bd598815a4145cff375233573'
|
7
|
+
data.tar.gz: 0abc8205f0f400f5cade71abe7f075a96338beb44c8559c79b5603f478ddbd8dec7dee4a7ca56784e2be2f10238f16667fd2d68a7e7e54b36caa0529fd62fd95
|
data/.circleci/config.yml
CHANGED
@@ -6,14 +6,14 @@ version: 2
|
|
6
6
|
jobs:
|
7
7
|
build:
|
8
8
|
docker:
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
9
|
+
- image: circleci/ruby:2.4.1-node-browsers
|
10
|
+
environment:
|
11
|
+
- CC_TEST_REPORTER_ID: 94ada9b95ee3f232a6e984809d37917cfee90ac47805429e8c49742b2e8d2276
|
12
|
+
- RAILS_ENV: test
|
13
|
+
- image: circleci/postgres:12.5-ram
|
14
|
+
environment:
|
15
|
+
- POSTGRES_HOST_AUTH_METHOD: trust
|
16
|
+
- POSTGRES_PASSWORD: password
|
17
17
|
|
18
18
|
working_directory: ~/repo
|
19
19
|
|
@@ -0,0 +1,30 @@
|
|
1
|
+
module Messaging
|
2
|
+
module Adapters
|
3
|
+
class Postgres
|
4
|
+
class Categories
|
5
|
+
class Row
|
6
|
+
extend Dry::Initializer
|
7
|
+
|
8
|
+
param :table_name
|
9
|
+
param :type
|
10
|
+
param :expression
|
11
|
+
|
12
|
+
def category_class
|
13
|
+
return CategoryWithPartitions if type == 'p'
|
14
|
+
|
15
|
+
Category
|
16
|
+
end
|
17
|
+
|
18
|
+
def category_name
|
19
|
+
regexp = /FOR VALUES IN \(\'(.*)\'\)/
|
20
|
+
expression.match(regexp)[1]
|
21
|
+
end
|
22
|
+
|
23
|
+
def to_category
|
24
|
+
category_class.new(category_name, table_name)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -5,21 +5,84 @@ module Messaging
|
|
5
5
|
include Enumerable
|
6
6
|
|
7
7
|
def each
|
8
|
-
|
9
|
-
|
10
|
-
all_categories.each do |name|
|
11
|
-
yield Category.new(name)
|
12
|
-
end
|
8
|
+
all_categories.each { |c| yield c }
|
13
9
|
end
|
14
10
|
|
11
|
+
# Get a category by name
|
12
|
+
#
|
13
|
+
# @param name [String] the name of the category
|
14
|
+
# @return [nil] if no category exists with the given name
|
15
|
+
# @return [Category]
|
15
16
|
def [](name)
|
16
|
-
|
17
|
+
all_categories.find { |c| c.name == name }
|
18
|
+
end
|
19
|
+
|
20
|
+
# Creates a table partition for the given category
|
21
|
+
#
|
22
|
+
# @param name [String] the name of the category
|
23
|
+
# @return [Category]
|
24
|
+
def create(name)
|
25
|
+
table_name = Category.table_name_for(name)
|
26
|
+
sql = <<~SQL
|
27
|
+
CREATE TABLE messaging.#{table_name}
|
28
|
+
PARTITION OF messaging.messages FOR VALUES IN ('#{name}');
|
29
|
+
SQL
|
30
|
+
connection.execute sql
|
31
|
+
Category.new(name, table_name)
|
32
|
+
end
|
33
|
+
|
34
|
+
# Creates a table partition for the given category
|
35
|
+
# that in turn is partitioned based on created_at
|
36
|
+
#
|
37
|
+
# @param name [String] the name of the category
|
38
|
+
# @return [CategoryWithPartitions]
|
39
|
+
def create_and_partition_by_day(name)
|
40
|
+
table_name = Category.table_name_for(name)
|
41
|
+
sql = <<~SQL
|
42
|
+
CREATE TABLE messaging.#{table_name}
|
43
|
+
PARTITION OF messaging.messages FOR VALUES IN ('#{name}')
|
44
|
+
PARTITION BY RANGE (created_at);
|
45
|
+
SQL
|
46
|
+
connection.execute sql
|
47
|
+
CategoryWithPartitions.new(name, table_name)
|
48
|
+
end
|
49
|
+
|
50
|
+
# Drops the table partition (including all messages) for the given category
|
51
|
+
#
|
52
|
+
# @param name [String] the name of the category
|
53
|
+
def drop(name)
|
54
|
+
table_name = Category.table_name_for(name)
|
55
|
+
sql = <<~SQL
|
56
|
+
drop TABLE messaging.#{table_name}
|
57
|
+
SQL
|
58
|
+
connection.execute sql
|
17
59
|
end
|
18
60
|
|
19
61
|
private
|
20
62
|
|
21
63
|
def all_categories
|
22
|
-
|
64
|
+
@all_categories ||= fetch_categories
|
65
|
+
end
|
66
|
+
|
67
|
+
def fetch_categories
|
68
|
+
sql = <<~SQL
|
69
|
+
SELECT child.relname AS category,
|
70
|
+
child.relkind AS category_type,
|
71
|
+
pg_get_expr(child.relpartbound, child.oid, true) AS expression
|
72
|
+
FROM pg_inherits
|
73
|
+
JOIN pg_class parent ON pg_inherits.inhparent = parent.oid
|
74
|
+
JOIN pg_class child ON pg_inherits.inhrelid = child.oid
|
75
|
+
JOIN pg_namespace ON pg_namespace.oid = parent.relnamespace
|
76
|
+
WHERE pg_namespace.nspname = 'messaging'
|
77
|
+
AND parent.relname = 'messages'
|
78
|
+
AND child.relkind in ('r', 'p')
|
79
|
+
ORDER BY category
|
80
|
+
SQL
|
81
|
+
connection.select_rows(sql).map { |r| Row.new(*r).to_category }
|
82
|
+
end
|
83
|
+
|
84
|
+
def connection
|
85
|
+
SerializedMessage.connection
|
23
86
|
end
|
24
87
|
end
|
25
88
|
end
|
@@ -2,16 +2,13 @@ module Messaging
|
|
2
2
|
module Adapters
|
3
3
|
class Postgres
|
4
4
|
class Category
|
5
|
-
|
6
|
-
attr_reader :name
|
5
|
+
extend Dry::Initializer
|
7
6
|
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
def initialize(name)
|
14
|
-
@name = name
|
7
|
+
param :name
|
8
|
+
param :table_name, default: -> { self.class.table_name_for(name) }
|
9
|
+
|
10
|
+
def self.table_name_for(name)
|
11
|
+
name.parameterize(separator: '_')
|
15
12
|
end
|
16
13
|
|
17
14
|
# Access to all messages in the category sorted by created_at
|
@@ -20,19 +17,8 @@ module Messaging
|
|
20
17
|
SerializedMessage.where(stream_category: name).order(:created_at)
|
21
18
|
end
|
22
19
|
|
23
|
-
def messages_older_than(time)
|
24
|
-
messages.where('created_at < ?', time)
|
25
|
-
end
|
26
|
-
|
27
|
-
def delete_messages_older_than!(time)
|
28
|
-
SerializedMessage.transaction do
|
29
|
-
ActiveRecord::Base.connection.execute "SET LOCAL statement_timeout = '0'"
|
30
|
-
messages_older_than(time).delete_all
|
31
|
-
end
|
32
|
-
end
|
33
|
-
|
34
20
|
def inspect
|
35
|
-
"#<Category
|
21
|
+
"#<Category: #{name}>"
|
36
22
|
end
|
37
23
|
end
|
38
24
|
end
|
@@ -0,0 +1,89 @@
|
|
1
|
+
module Messaging
|
2
|
+
module Adapters
|
3
|
+
class Postgres
|
4
|
+
class CategoryWithPartitions < Category
|
5
|
+
def add_partition(date:)
|
6
|
+
from, to = partition_range_for(date: date)
|
7
|
+
partition_name = partition_name_for(date: from)
|
8
|
+
|
9
|
+
return if partition_exists?(partition_name)
|
10
|
+
|
11
|
+
sql = <<~SQL
|
12
|
+
CREATE TABLE IF NOT EXISTS messaging.#{partition_name}
|
13
|
+
PARTITION OF messaging.#{table_name} FOR VALUES FROM ('#{from}') TO ('#{to}')
|
14
|
+
SQL
|
15
|
+
|
16
|
+
SerializedMessage.transaction do
|
17
|
+
AdvisoryTransactionLock.call key: partition_name
|
18
|
+
SerializedMessage.connection.execute sql
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
# Creates multiple partitions
|
23
|
+
#
|
24
|
+
# @param start_date [Date] from which date to create partitions
|
25
|
+
# @param days [Integer] how many days worth of partitions to create
|
26
|
+
def add_partitions(start_date:, days: 1)
|
27
|
+
first = start_date.to_date
|
28
|
+
last = first + (days - 1)
|
29
|
+
|
30
|
+
(first..last).each do |date|
|
31
|
+
add_partition(date: date)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
# Removes a partition including the included messages
|
36
|
+
#
|
37
|
+
# @param partition_name [String] the name of the partition to drop
|
38
|
+
def drop_partition(partition_name)
|
39
|
+
return unless partition_name.match?(/^#{table_name}_\d{4}_\d{2}_\d{2}$/)
|
40
|
+
|
41
|
+
SerializedMessage.connection.execute "drop TABLE messaging.#{partition_name}"
|
42
|
+
end
|
43
|
+
|
44
|
+
# Removes all partitions older than the given date
|
45
|
+
#
|
46
|
+
# @param date [Date] the cutoff date
|
47
|
+
def drop_partitions_older_than(date)
|
48
|
+
max_partition_name = partition_name_for(date: date)
|
49
|
+
partitions.select { |p| p < max_partition_name }.each do |p|
|
50
|
+
drop_partition(p)
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def partition_name_for(date:)
|
55
|
+
"#{name}_%d_%02d_%02d" % [date.year, date.month, date.day]
|
56
|
+
end
|
57
|
+
|
58
|
+
def partition_range_for(date:)
|
59
|
+
from = date.to_time.beginning_of_day
|
60
|
+
to = from + 1.day
|
61
|
+
[from, to]
|
62
|
+
end
|
63
|
+
|
64
|
+
def partition_exists?(partition_name)
|
65
|
+
partitions.include? partition_name
|
66
|
+
end
|
67
|
+
|
68
|
+
def partitions
|
69
|
+
sql = <<~SQL
|
70
|
+
SELECT child.relname AS name
|
71
|
+
FROM pg_inherits
|
72
|
+
JOIN pg_class parent ON pg_inherits.inhparent = parent.oid
|
73
|
+
JOIN pg_class child ON pg_inherits.inhrelid = child.oid
|
74
|
+
JOIN pg_namespace ON pg_namespace.oid = parent.relnamespace
|
75
|
+
WHERE pg_namespace.nspname = 'messaging'
|
76
|
+
AND parent.relname = '#{table_name}'
|
77
|
+
AND child.relkind = 'r'
|
78
|
+
ORDER BY name
|
79
|
+
SQL
|
80
|
+
SerializedMessage.connection.select_values(sql)
|
81
|
+
end
|
82
|
+
|
83
|
+
def inspect
|
84
|
+
"#<CategoryWithPartitions: #{name}>"
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
@@ -1,5 +1,7 @@
|
|
1
1
|
require_relative 'category'
|
2
|
+
require_relative 'category_with_partitions'
|
2
3
|
require_relative 'categories'
|
4
|
+
require_relative 'categories/row'
|
3
5
|
require_relative 'stream'
|
4
6
|
require_relative 'streams'
|
5
7
|
|
@@ -22,16 +24,11 @@ module Messaging
|
|
22
24
|
# @see Streams
|
23
25
|
attr_reader :streams
|
24
26
|
|
25
|
-
# @return [Categories] all the stream categories in the store
|
26
|
-
# @see Categories
|
27
|
-
attr_reader :categories
|
28
|
-
|
29
27
|
# Should not be used directly. Access the store though
|
30
28
|
# Messaging.message_store or Messaging::Adapters::Store[:postgres]
|
31
29
|
# @api private
|
32
30
|
def initialize
|
33
31
|
@streams = Streams.new
|
34
|
-
@categories = Categories.new
|
35
32
|
end
|
36
33
|
|
37
34
|
# Get a specific stream by name
|
@@ -41,8 +38,14 @@ module Messaging
|
|
41
38
|
streams[name]
|
42
39
|
end
|
43
40
|
|
41
|
+
# @return [Categories] all the stream categories in the store
|
42
|
+
# @see Categories
|
43
|
+
def categories
|
44
|
+
Categories.new
|
45
|
+
end
|
46
|
+
|
44
47
|
# Get a specific category by name
|
45
|
-
# @return [
|
48
|
+
# @return [Category]
|
46
49
|
# @see Messaging.category
|
47
50
|
def category(name)
|
48
51
|
categories[name]
|
@@ -82,6 +85,13 @@ module Messaging
|
|
82
85
|
return message unless message.stream_name
|
83
86
|
|
84
87
|
SerializedMessage.create!(message: message).to_message
|
88
|
+
rescue ActiveRecord::StatementInvalid => e
|
89
|
+
category = message.category
|
90
|
+
raise e unless e.message.include?('no partition of relation')
|
91
|
+
raise e unless category || category.is_a?(CategoryWithPartitions)
|
92
|
+
|
93
|
+
category.add_partition(date: Date.today)
|
94
|
+
retry
|
85
95
|
end
|
86
96
|
end
|
87
97
|
end
|
@@ -18,6 +18,38 @@ module Messaging
|
|
18
18
|
private_class_method :register!
|
19
19
|
|
20
20
|
register!
|
21
|
+
|
22
|
+
def create_messages_table
|
23
|
+
sql = <<~SQL
|
24
|
+
CREATE SCHEMA IF NOT EXISTS messaging;
|
25
|
+
CREATE SEQUENCE IF NOT EXISTS messaging.messages_id_seq;
|
26
|
+
|
27
|
+
CREATE TABLE messaging.messages (
|
28
|
+
id bigint DEFAULT nextval('messaging.messages_id_seq'::regclass) NOT NULL,
|
29
|
+
uuid uuid NOT NULL,
|
30
|
+
stream character varying NOT NULL,
|
31
|
+
stream_position bigint NOT NULL,
|
32
|
+
message_type character varying NOT NULL,
|
33
|
+
data jsonb,
|
34
|
+
created_at timestamp without time zone NOT NULL,
|
35
|
+
updated_at timestamp without time zone NOT NULL,
|
36
|
+
stream_category character varying,
|
37
|
+
stream_id character varying
|
38
|
+
)
|
39
|
+
PARTITION BY LIST (stream_category);
|
40
|
+
|
41
|
+
CREATE INDEX messages_id_idx ON ONLY messaging.messages USING btree (id);
|
42
|
+
CREATE INDEX messages_stream_category_id_idx ON ONLY messaging.messages USING btree (stream_category, id);
|
43
|
+
CREATE INDEX messages_stream_category_stream_id_stream_position_idx ON ONLY messaging.messages USING btree (stream_category, stream_id, stream_position);
|
44
|
+
SQL
|
45
|
+
connection.execute sql
|
46
|
+
end
|
47
|
+
|
48
|
+
private
|
49
|
+
|
50
|
+
def connection
|
51
|
+
ActiveRecord::Base.connection
|
52
|
+
end
|
21
53
|
end
|
22
54
|
end
|
23
55
|
end
|
data/lib/messaging/message.rb
CHANGED
data/lib/messaging/version.rb
CHANGED
data/lib/messaging.rb
CHANGED
@@ -67,6 +67,17 @@ module Messaging
|
|
67
67
|
result
|
68
68
|
end
|
69
69
|
|
70
|
+
# Access the stream categories in the current message store
|
71
|
+
#
|
72
|
+
# @example Creating a new category
|
73
|
+
# Messaging.categories.create('customer')
|
74
|
+
#
|
75
|
+
# @return [Messaging::Adapters::Test::Categories] when using the test adapter
|
76
|
+
# @return [Messaging::Adapters::Postgres::Categories] when using the postgres adapter
|
77
|
+
def self.categories
|
78
|
+
message_store.categories
|
79
|
+
end
|
80
|
+
|
70
81
|
def self.category(name)
|
71
82
|
message_store.category(name)
|
72
83
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: messaging
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.7.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Bukowskis
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-02-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activerecord
|
@@ -273,7 +273,6 @@ files:
|
|
273
273
|
- ".circleci/config.yml"
|
274
274
|
- ".gitignore"
|
275
275
|
- ".rspec"
|
276
|
-
- ".ruby-version"
|
277
276
|
- Gemfile
|
278
277
|
- Gemfile.lock
|
279
278
|
- README.md
|
@@ -298,7 +297,9 @@ files:
|
|
298
297
|
- lib/messaging/adapters/postgres.rb
|
299
298
|
- lib/messaging/adapters/postgres/advisory_transaction_lock.rb
|
300
299
|
- lib/messaging/adapters/postgres/categories.rb
|
300
|
+
- lib/messaging/adapters/postgres/categories/row.rb
|
301
301
|
- lib/messaging/adapters/postgres/category.rb
|
302
|
+
- lib/messaging/adapters/postgres/category_with_partitions.rb
|
302
303
|
- lib/messaging/adapters/postgres/serialized_message.rb
|
303
304
|
- lib/messaging/adapters/postgres/store.rb
|
304
305
|
- lib/messaging/adapters/postgres/stream.rb
|
@@ -351,7 +352,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
351
352
|
- !ruby/object:Gem::Version
|
352
353
|
version: '0'
|
353
354
|
requirements: []
|
354
|
-
rubygems_version: 3.1
|
355
|
+
rubygems_version: 3.0.3.1
|
355
356
|
signing_key:
|
356
357
|
specification_version: 4
|
357
358
|
summary: A library for decoupling applications by using messaging to communicate between
|
data/.ruby-version
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
2.3.1
|