messaging 3.5.7 → 3.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 44057f3b83334d34fb1cefa4b0261d84ef93bf764c487ae7084c425dac105643
4
- data.tar.gz: 38a69af6df0a7d1b86f492d21d3b4ffe3e0d2e4ff360fe53cc106f499da6879f
3
+ metadata.gz: 860c436d2faab463d40144358859eb282aa352370ffb2c725cfa292c267eb959
4
+ data.tar.gz: cf39e2bf8a10ebe84cfe09d9e74d0fa9bce654803478865b47601b5ada04d404
5
5
  SHA512:
6
- metadata.gz: c1119c1db6d97fe860cfd82dcb9547546b5313caf174557756453a0aa354823a335a306784e35c022d2f0a693057707789c56fa34369e58c81f0555ad457ce4b
7
- data.tar.gz: 90640e22a90b03de047daf4a228c81defa88f56321eca71f725ad898c9dae182edab6e7523408bde73d9f1fb26a3ce1efe08db9b298d96500f4f1967c963d461
6
+ metadata.gz: '0751658ec0a77cc8fd925963c1530c05ee31cb458c8f773a8aba7dd333569e90547945d52dda5cbee21e0f0668cf6d7909cb2f2bd598815a4145cff375233573'
7
+ data.tar.gz: 0abc8205f0f400f5cade71abe7f075a96338beb44c8559c79b5603f478ddbd8dec7dee4a7ca56784e2be2f10238f16667fd2d68a7e7e54b36caa0529fd62fd95
data/.circleci/config.yml CHANGED
@@ -6,14 +6,14 @@ version: 2
6
6
  jobs:
7
7
  build:
8
8
  docker:
9
- # specify the version you desire here
10
- - image: circleci/ruby:2.4.1-node-browsers
11
- environment:
12
- CC_TEST_REPORTER_ID: 94ada9b95ee3f232a6e984809d37917cfee90ac47805429e8c49742b2e8d2276
13
- RAILS_ENV: test
14
- - image: circleci/postgres:9.6.6-alpine
15
- environment:
16
- POSTGRES_USER: postgres
9
+ - image: circleci/ruby:2.4.1-node-browsers
10
+ environment:
11
+ - CC_TEST_REPORTER_ID: 94ada9b95ee3f232a6e984809d37917cfee90ac47805429e8c49742b2e8d2276
12
+ - RAILS_ENV: test
13
+ - image: circleci/postgres:12.5-ram
14
+ environment:
15
+ - POSTGRES_HOST_AUTH_METHOD: trust
16
+ - POSTGRES_PASSWORD: password
17
17
 
18
18
  working_directory: ~/repo
19
19
 
@@ -44,7 +44,7 @@ module Messaging
44
44
  end
45
45
 
46
46
  def create_producer
47
- kafka.client.async_producer(Config.kafka.producer.to_hash)
47
+ kafka.client.async_producer(Config.kafka.producer.to_h)
48
48
  end
49
49
  end
50
50
  end
@@ -0,0 +1,30 @@
1
+ module Messaging
2
+ module Adapters
3
+ class Postgres
4
+ class Categories
5
+ class Row
6
+ extend Dry::Initializer
7
+
8
+ param :table_name
9
+ param :type
10
+ param :expression
11
+
12
+ def category_class
13
+ return CategoryWithPartitions if type == 'p'
14
+
15
+ Category
16
+ end
17
+
18
+ def category_name
19
+ regexp = /FOR VALUES IN \(\'(.*)\'\)/
20
+ expression.match(regexp)[1]
21
+ end
22
+
23
+ def to_category
24
+ category_class.new(category_name, table_name)
25
+ end
26
+ end
27
+ end
28
+ end
29
+ end
30
+ end
@@ -5,21 +5,84 @@ module Messaging
5
5
  include Enumerable
6
6
 
7
7
  def each
8
- return enum_for(:each) unless block_given?
9
-
10
- all_categories.each do |name|
11
- yield Category.new(name)
12
- end
8
+ all_categories.each { |c| yield c }
13
9
  end
14
10
 
11
+ # Get a category by name
12
+ #
13
+ # @param name [String] the name of the category
14
+ # @return [nil] if no category exists with the given name
15
+ # @return [Category]
15
16
  def [](name)
16
- Category.new(name)
17
+ all_categories.find { |c| c.name == name }
18
+ end
19
+
20
+ # Creates a table partition for the given category
21
+ #
22
+ # @param name [String] the name of the category
23
+ # @return [Category]
24
+ def create(name)
25
+ table_name = Category.table_name_for(name)
26
+ sql = <<~SQL
27
+ CREATE TABLE messaging.#{table_name}
28
+ PARTITION OF messaging.messages FOR VALUES IN ('#{name}');
29
+ SQL
30
+ connection.execute sql
31
+ Category.new(name, table_name)
32
+ end
33
+
34
+ # Creates a table partition for the given category
35
+ # that in turn is partitioned based on created_at
36
+ #
37
+ # @param name [String] the name of the category
38
+ # @return [CategoryWithPartitions]
39
+ def create_and_partition_by_day(name)
40
+ table_name = Category.table_name_for(name)
41
+ sql = <<~SQL
42
+ CREATE TABLE messaging.#{table_name}
43
+ PARTITION OF messaging.messages FOR VALUES IN ('#{name}')
44
+ PARTITION BY RANGE (created_at);
45
+ SQL
46
+ connection.execute sql
47
+ CategoryWithPartitions.new(name, table_name)
48
+ end
49
+
50
+ # Drops the table partition (including all messages) for the given category
51
+ #
52
+ # @param name [String] the name of the category
53
+ def drop(name)
54
+ table_name = Category.table_name_for(name)
55
+ sql = <<~SQL
56
+ drop TABLE messaging.#{table_name}
57
+ SQL
58
+ connection.execute sql
17
59
  end
18
60
 
19
61
  private
20
62
 
21
63
  def all_categories
22
- SerializedMessage.distinct.pluck(:stream_category).lazy
64
+ @all_categories ||= fetch_categories
65
+ end
66
+
67
+ def fetch_categories
68
+ sql = <<~SQL
69
+ SELECT child.relname AS category,
70
+ child.relkind AS category_type,
71
+ pg_get_expr(child.relpartbound, child.oid, true) AS expression
72
+ FROM pg_inherits
73
+ JOIN pg_class parent ON pg_inherits.inhparent = parent.oid
74
+ JOIN pg_class child ON pg_inherits.inhrelid = child.oid
75
+ JOIN pg_namespace ON pg_namespace.oid = parent.relnamespace
76
+ WHERE pg_namespace.nspname = 'messaging'
77
+ AND parent.relname = 'messages'
78
+ AND child.relkind in ('r', 'p')
79
+ ORDER BY category
80
+ SQL
81
+ connection.select_rows(sql).map { |r| Row.new(*r).to_category }
82
+ end
83
+
84
+ def connection
85
+ SerializedMessage.connection
23
86
  end
24
87
  end
25
88
  end
@@ -2,16 +2,13 @@ module Messaging
2
2
  module Adapters
3
3
  class Postgres
4
4
  class Category
5
- # @return [String] the name of the category
6
- attr_reader :name
5
+ extend Dry::Initializer
7
6
 
8
- # Should not be used directly.
9
- # Use {Messaging.category} or {Store#category}
10
- # @api private
11
- # @see Messaging.category
12
- # @see Store.category
13
- def initialize(name)
14
- @name = name
7
+ param :name
8
+ param :table_name, default: -> { self.class.table_name_for(name) }
9
+
10
+ def self.table_name_for(name)
11
+ name.parameterize(separator: '_')
15
12
  end
16
13
 
17
14
  # Access to all messages in the category sorted by created_at
@@ -20,19 +17,8 @@ module Messaging
20
17
  SerializedMessage.where(stream_category: name).order(:created_at)
21
18
  end
22
19
 
23
- def messages_older_than(time)
24
- messages.where('created_at < ?', time)
25
- end
26
-
27
- def delete_messages_older_than!(time)
28
- SerializedMessage.transaction do
29
- ActiveRecord::Base.connection.execute "SET LOCAL statement_timeout = '0'"
30
- messages_older_than(time).delete_all
31
- end
32
- end
33
-
34
20
  def inspect
35
- "#<Category:#{name}>>"
21
+ "#<Category: #{name}>"
36
22
  end
37
23
  end
38
24
  end
@@ -0,0 +1,89 @@
1
+ module Messaging
2
+ module Adapters
3
+ class Postgres
4
+ class CategoryWithPartitions < Category
5
+ def add_partition(date:)
6
+ from, to = partition_range_for(date: date)
7
+ partition_name = partition_name_for(date: from)
8
+
9
+ return if partition_exists?(partition_name)
10
+
11
+ sql = <<~SQL
12
+ CREATE TABLE IF NOT EXISTS messaging.#{partition_name}
13
+ PARTITION OF messaging.#{table_name} FOR VALUES FROM ('#{from}') TO ('#{to}')
14
+ SQL
15
+
16
+ SerializedMessage.transaction do
17
+ AdvisoryTransactionLock.call key: partition_name
18
+ SerializedMessage.connection.execute sql
19
+ end
20
+ end
21
+
22
+ # Creates multiple partitions
23
+ #
24
+ # @param start_date [Date] from which date to create partitions
25
+ # @param days [Integer] how many days worth of partitions to create
26
+ def add_partitions(start_date:, days: 1)
27
+ first = start_date.to_date
28
+ last = first + (days - 1)
29
+
30
+ (first..last).each do |date|
31
+ add_partition(date: date)
32
+ end
33
+ end
34
+
35
+ # Removes a partition including the included messages
36
+ #
37
+ # @param partition_name [String] the name of the partition to drop
38
+ def drop_partition(partition_name)
39
+ return unless partition_name.match?(/^#{table_name}_\d{4}_\d{2}_\d{2}$/)
40
+
41
+ SerializedMessage.connection.execute "drop TABLE messaging.#{partition_name}"
42
+ end
43
+
44
+ # Removes all partitions older than the given date
45
+ #
46
+ # @param date [Date] the cutoff date
47
+ def drop_partitions_older_than(date)
48
+ max_partition_name = partition_name_for(date: date)
49
+ partitions.select { |p| p < max_partition_name }.each do |p|
50
+ drop_partition(p)
51
+ end
52
+ end
53
+
54
+ def partition_name_for(date:)
55
+ "#{name}_%d_%02d_%02d" % [date.year, date.month, date.day]
56
+ end
57
+
58
+ def partition_range_for(date:)
59
+ from = date.to_time.beginning_of_day
60
+ to = from + 1.day
61
+ [from, to]
62
+ end
63
+
64
+ def partition_exists?(partition_name)
65
+ partitions.include? partition_name
66
+ end
67
+
68
+ def partitions
69
+ sql = <<~SQL
70
+ SELECT child.relname AS name
71
+ FROM pg_inherits
72
+ JOIN pg_class parent ON pg_inherits.inhparent = parent.oid
73
+ JOIN pg_class child ON pg_inherits.inhrelid = child.oid
74
+ JOIN pg_namespace ON pg_namespace.oid = parent.relnamespace
75
+ WHERE pg_namespace.nspname = 'messaging'
76
+ AND parent.relname = '#{table_name}'
77
+ AND child.relkind = 'r'
78
+ ORDER BY name
79
+ SQL
80
+ SerializedMessage.connection.select_values(sql)
81
+ end
82
+
83
+ def inspect
84
+ "#<CategoryWithPartitions: #{name}>"
85
+ end
86
+ end
87
+ end
88
+ end
89
+ end
@@ -2,7 +2,7 @@ module Messaging
2
2
  module Adapters
3
3
  class Postgres
4
4
  class SerializedMessage < ActiveRecord::Base
5
- self.table_name = :messaging_messages
5
+ self.table_name = 'messaging.messages'
6
6
 
7
7
  attr_accessor :expected_version
8
8
 
@@ -1,5 +1,7 @@
1
1
  require_relative 'category'
2
+ require_relative 'category_with_partitions'
2
3
  require_relative 'categories'
4
+ require_relative 'categories/row'
3
5
  require_relative 'stream'
4
6
  require_relative 'streams'
5
7
 
@@ -22,16 +24,11 @@ module Messaging
22
24
  # @see Streams
23
25
  attr_reader :streams
24
26
 
25
- # @return [Categories] all the stream categories in the store
26
- # @see Categories
27
- attr_reader :categories
28
-
29
27
  # Should not be used directly. Access the store though
30
28
  # Messaging.message_store or Messaging::Adapters::Store[:postgres]
31
29
  # @api private
32
30
  def initialize
33
31
  @streams = Streams.new
34
- @categories = Categories.new
35
32
  end
36
33
 
37
34
  # Get a specific stream by name
@@ -41,8 +38,14 @@ module Messaging
41
38
  streams[name]
42
39
  end
43
40
 
41
+ # @return [Categories] all the stream categories in the store
42
+ # @see Categories
43
+ def categories
44
+ Categories.new
45
+ end
46
+
44
47
  # Get a specific category by name
45
- # @return [Stream]
48
+ # @return [Category]
46
49
  # @see Messaging.category
47
50
  def category(name)
48
51
  categories[name]
@@ -82,6 +85,13 @@ module Messaging
82
85
  return message unless message.stream_name
83
86
 
84
87
  SerializedMessage.create!(message: message).to_message
88
+ rescue ActiveRecord::StatementInvalid => e
89
+ category = message.category
90
+ raise e unless e.message.include?('no partition of relation')
91
+ raise e unless category || category.is_a?(CategoryWithPartitions)
92
+
93
+ category.add_partition(date: Date.today)
94
+ retry
85
95
  end
86
96
  end
87
97
  end
@@ -18,6 +18,38 @@ module Messaging
18
18
  private_class_method :register!
19
19
 
20
20
  register!
21
+
22
+ def create_messages_table
23
+ sql = <<~SQL
24
+ CREATE SCHEMA IF NOT EXISTS messaging;
25
+ CREATE SEQUENCE IF NOT EXISTS messaging.messages_id_seq;
26
+
27
+ CREATE TABLE messaging.messages (
28
+ id bigint DEFAULT nextval('messaging.messages_id_seq'::regclass) NOT NULL,
29
+ uuid uuid NOT NULL,
30
+ stream character varying NOT NULL,
31
+ stream_position bigint NOT NULL,
32
+ message_type character varying NOT NULL,
33
+ data jsonb,
34
+ created_at timestamp without time zone NOT NULL,
35
+ updated_at timestamp without time zone NOT NULL,
36
+ stream_category character varying,
37
+ stream_id character varying
38
+ )
39
+ PARTITION BY LIST (stream_category);
40
+
41
+ CREATE INDEX messages_id_idx ON ONLY messaging.messages USING btree (id);
42
+ CREATE INDEX messages_stream_category_id_idx ON ONLY messaging.messages USING btree (stream_category, id);
43
+ CREATE INDEX messages_stream_category_stream_id_stream_position_idx ON ONLY messaging.messages USING btree (stream_category, stream_id, stream_position);
44
+ SQL
45
+ connection.execute sql
46
+ end
47
+
48
+ private
49
+
50
+ def connection
51
+ ActiveRecord::Base.connection
52
+ end
21
53
  end
22
54
  end
23
55
  end
@@ -11,10 +11,6 @@ module Messaging
11
11
  def messages
12
12
  @messages ||= []
13
13
  end
14
-
15
- def delete_messages_older_than!(time)
16
- messages.delete_if { |m| m.timestamp < time }
17
- end
18
14
  end
19
15
  end
20
16
  end
@@ -124,6 +124,10 @@ module Messaging
124
124
  stream_name.split('$').first
125
125
  end
126
126
 
127
+ def category
128
+ Messaging.category(stream_category)
129
+ end
130
+
127
131
  def stream_id
128
132
  return unless stream_name
129
133
 
@@ -1,3 +1,3 @@
1
1
  module Messaging
2
- VERSION = '3.5.7'.freeze
2
+ VERSION = '3.7.0'.freeze
3
3
  end
data/lib/messaging.rb CHANGED
@@ -67,6 +67,17 @@ module Messaging
67
67
  result
68
68
  end
69
69
 
70
+ # Access the stream categories in the current message store
71
+ #
72
+ # @example Creating a new category
73
+ # Messaging.categories.create('customer')
74
+ #
75
+ # @return [Messaging::Adapters::Test::Categories] when using the test adapter
76
+ # @return [Messaging::Adapters::Postgres::Categories] when using the postgres adapter
77
+ def self.categories
78
+ message_store.categories
79
+ end
80
+
70
81
  def self.category(name)
71
82
  message_store.category(name)
72
83
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: messaging
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.5.7
4
+ version: 3.7.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Bukowskis
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2021-05-31 00:00:00.000000000 Z
11
+ date: 2022-02-07 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activerecord
@@ -273,7 +273,6 @@ files:
273
273
  - ".circleci/config.yml"
274
274
  - ".gitignore"
275
275
  - ".rspec"
276
- - ".ruby-version"
277
276
  - Gemfile
278
277
  - Gemfile.lock
279
278
  - README.md
@@ -298,7 +297,9 @@ files:
298
297
  - lib/messaging/adapters/postgres.rb
299
298
  - lib/messaging/adapters/postgres/advisory_transaction_lock.rb
300
299
  - lib/messaging/adapters/postgres/categories.rb
300
+ - lib/messaging/adapters/postgres/categories/row.rb
301
301
  - lib/messaging/adapters/postgres/category.rb
302
+ - lib/messaging/adapters/postgres/category_with_partitions.rb
302
303
  - lib/messaging/adapters/postgres/serialized_message.rb
303
304
  - lib/messaging/adapters/postgres/store.rb
304
305
  - lib/messaging/adapters/postgres/stream.rb
@@ -351,7 +352,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
351
352
  - !ruby/object:Gem::Version
352
353
  version: '0'
353
354
  requirements: []
354
- rubygems_version: 3.1.4
355
+ rubygems_version: 3.0.3.1
355
356
  signing_key:
356
357
  specification_version: 4
357
358
  summary: A library for decoupling applications by using messaging to communicate between
data/.ruby-version DELETED
@@ -1 +0,0 @@
1
- 2.3.1