messaging 3.6.2 → 3.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '049d3a3eac021195a546e67f3ce5351fa93a105a86a7bddc10b6fcb140533e43'
-  data.tar.gz: bce77446512ba9f31d3b21739ef2af2c59ac8bd89c2ff7d15006459aff3cd387
+  metadata.gz: 860c436d2faab463d40144358859eb282aa352370ffb2c725cfa292c267eb959
+  data.tar.gz: cf39e2bf8a10ebe84cfe09d9e74d0fa9bce654803478865b47601b5ada04d404
 SHA512:
-  metadata.gz: 2b54e3f6633fa243b6fba4d20e2359c8018cbb37634a7420c75f9e2e8cf6a9da44aa0bebc139ed06ded0b887a35383be72c38c8a58ee6f3e6d288d5527b73e28
-  data.tar.gz: f45a5fb8cee52d26d0ef769a5557ac2fcabebc0b27bc5374a2bd5b25c930a3731b80238efc22429290bfdb0be14b915e2fc995af299d2580b1a6f2160462512f
+  metadata.gz: '0751658ec0a77cc8fd925963c1530c05ee31cb458c8f773a8aba7dd333569e90547945d52dda5cbee21e0f0668cf6d7909cb2f2bd598815a4145cff375233573'
+  data.tar.gz: 0abc8205f0f400f5cade71abe7f075a96338beb44c8559c79b5603f478ddbd8dec7dee4a7ca56784e2be2f10238f16667fd2d68a7e7e54b36caa0529fd62fd95

data/lib/messaging/adapters/postgres/categories/row.rb

@@ -0,0 +1,30 @@
+module Messaging
+  module Adapters
+    class Postgres
+      class Categories
+        class Row
+          extend Dry::Initializer
+
+          param :table_name
+          param :type
+          param :expression
+
+          def category_class
+            return CategoryWithPartitions if type == 'p'
+
+            Category
+          end
+
+          def category_name
+            regexp = /FOR VALUES IN \(\'(.*)\'\)/
+            expression.match(regexp)[1]
+          end
+
+          def to_category
+            category_class.new(category_name, table_name)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/messaging/adapters/postgres/categories.rb

@@ -5,9 +5,7 @@ module Messaging
         include Enumerable
 
         def each
-          all_categories.each do |name|
-            yield Category.new(name)
-          end
+          all_categories.each { |c| yield c }
         end
 
         # Get a category by name
@@ -16,10 +14,7 @@ module Messaging
         # @return [nil] if no category exists with the given name
         # @return [Category]
         def [](name)
-          category = all_categories.select { |c| c == name }
-          return unless category
-
-          Category.new(category)
+          all_categories.find { |c| c.name == name }
         end
 
         # Creates a table partition for the given category
@@ -27,20 +22,36 @@ module Messaging
         # @param name [String] the name of the category
         # @return [Category]
         def create(name)
-          table_name = name.parameterize(separator: '_')
+          table_name = Category.table_name_for(name)
           sql = <<~SQL
             CREATE TABLE messaging.#{table_name}
             PARTITION OF messaging.messages FOR VALUES IN ('#{name}');
           SQL
           connection.execute sql
-          Category.new(name)
+          Category.new(name, table_name)
+        end
+
+        # Creates a table partition for the given category
+        # that in turn is partitioned based on created_at
+        #
+        # @param name [String] the name of the category
+        # @return [CategoryWithPartitions]
+        def create_and_partition_by_day(name)
+          table_name = Category.table_name_for(name)
+          sql = <<~SQL
+            CREATE TABLE messaging.#{table_name}
+            PARTITION OF messaging.messages FOR VALUES IN ('#{name}')
+            PARTITION BY RANGE (created_at);
+          SQL
+          connection.execute sql
+          CategoryWithPartitions.new(name, table_name)
         end
 
         # Drops the table partition (including all messages) for the given category
         #
         # @param name [String] the name of the category
         def drop(name)
-          table_name = name.parameterize(separator: '_')
+          table_name = Category.table_name_for(name)
           sql = <<~SQL
             drop TABLE messaging.#{table_name}
           SQL
@@ -55,15 +66,19 @@ module Messaging
 
         def fetch_categories
           sql = <<~SQL
-            SELECT child.relname AS category
+            SELECT child.relname AS category,
+                   child.relkind AS category_type,
+                   pg_get_expr(child.relpartbound, child.oid, true) AS expression
             FROM pg_inherits
             JOIN pg_class parent ON pg_inherits.inhparent = parent.oid
             JOIN pg_class child  ON pg_inherits.inhrelid   = child.oid
             JOIN pg_namespace ON pg_namespace.oid  = parent.relnamespace
-            WHERE pg_namespace.nspname = 'messaging' AND child.relkind in ('r', 'p')
+            WHERE pg_namespace.nspname = 'messaging'
+            AND parent.relname = 'messages'
+            AND child.relkind in ('r', 'p')
             ORDER BY category
           SQL
-          connection.select_values(sql)
+          connection.select_rows(sql).map { |r| Row.new(*r).to_category }
         end
 
         def connection

data/lib/messaging/adapters/postgres/category.rb

@@ -2,16 +2,13 @@ module Messaging
   module Adapters
     class Postgres
       class Category
-        # @return [String] the name of the category
-        attr_reader :name
+        extend Dry::Initializer
 
-        # Should not be used directly.
-        # Use {Messaging.category} or {Store#category}
-        # @api private
-        # @see Messaging.category
-        # @see Store.category
-        def initialize(name)
-          @name = name
+        param :name
+        param :table_name, default: -> { self.class.table_name_for(name) }
+
+        def self.table_name_for(name)
+          name.parameterize(separator: '_')
         end
 
         # Access to all messages in the category sorted by created_at
@@ -20,19 +17,8 @@ module Messaging
           SerializedMessage.where(stream_category: name).order(:created_at)
         end
 
-        def messages_older_than(time)
-          messages.where('created_at < ?', time)
-        end
-
-        def delete_messages_older_than!(time)
-          SerializedMessage.transaction do
-            ActiveRecord::Base.connection.execute "SET LOCAL statement_timeout = '0'"
-            messages_older_than(time).delete_all
-          end
-        end
-
         def inspect
-          "#<Category:#{name}>>"
+          "#<Category: #{name}>"
         end
       end
     end

data/lib/messaging/adapters/postgres/category_with_partitions.rb

@@ -0,0 +1,89 @@
+module Messaging
+  module Adapters
+    class Postgres
+      class CategoryWithPartitions < Category
+        def add_partition(date:)
+          from, to = partition_range_for(date: date)
+          partition_name = partition_name_for(date: from)
+
+          return if partition_exists?(partition_name)
+
+          sql = <<~SQL
+            CREATE TABLE IF NOT EXISTS messaging.#{partition_name}
+            PARTITION OF messaging.#{table_name} FOR VALUES FROM ('#{from}') TO ('#{to}')
+          SQL
+
+          SerializedMessage.transaction do
+            AdvisoryTransactionLock.call key: partition_name
+            SerializedMessage.connection.execute sql
+          end
+        end
+
+        # Creates multiple partitions
+        #
+        # @param start_date [Date] from which date to create partitions
+        # @param days [Integer] how many days worth of partitions to create
+        def add_partitions(start_date:, days: 1)
+          first = start_date.to_date
+          last = first + (days - 1)
+
+          (first..last).each do |date|
+            add_partition(date: date)
+          end
+        end
+
+        # Removes a partition including the included messages
+        #
+        # @param partition_name [String] the name of the partition to drop
+        def drop_partition(partition_name)
+          return unless partition_name.match?(/^#{table_name}_\d{4}_\d{2}_\d{2}$/)
+
+          SerializedMessage.connection.execute "drop TABLE messaging.#{partition_name}"
+        end
+
+        # Removes all partitions older than the given date
+        #
+        # @param date [Date] the cutoff date
+        def drop_partitions_older_than(date)
+          max_partition_name = partition_name_for(date: date)
+          partitions.select { |p| p < max_partition_name }.each do |p|
+            drop_partition(p)
+          end
+        end
+
+        def partition_name_for(date:)
+          "#{name}_%d_%02d_%02d" % [date.year, date.month, date.day]
+        end
+
+        def partition_range_for(date:)
+          from = date.to_time.beginning_of_day
+          to = from + 1.day
+          [from, to]
+        end
+
+        def partition_exists?(partition_name)
+          partitions.include? partition_name
+        end
+
+        def partitions
+          sql = <<~SQL
+            SELECT child.relname AS name
+            FROM pg_inherits
+            JOIN pg_class parent ON pg_inherits.inhparent = parent.oid
+            JOIN pg_class child  ON pg_inherits.inhrelid   = child.oid
+            JOIN pg_namespace ON pg_namespace.oid  = parent.relnamespace
+            WHERE pg_namespace.nspname = 'messaging'
+            AND parent.relname = '#{table_name}'
+            AND child.relkind = 'r'
+            ORDER BY name
+          SQL
+          SerializedMessage.connection.select_values(sql)
+        end
+
+        def inspect
+          "#<CategoryWithPartitions: #{name}>"
+        end
+      end
+    end
+  end
+end

data/lib/messaging/adapters/postgres/store.rb

@@ -1,5 +1,7 @@
 require_relative 'category'
+require_relative 'category_with_partitions'
 require_relative 'categories'
+require_relative 'categories/row'
 require_relative 'stream'
 require_relative 'streams'
 
@@ -83,6 +85,13 @@ module Messaging
           return message unless message.stream_name
 
           SerializedMessage.create!(message: message).to_message
+        rescue ActiveRecord::StatementInvalid => e
+          category = message.category
+          raise e unless e.message.include?('no partition of relation')
+          raise e unless category || category.is_a?(CategoryWithPartitions)
+
+          category.add_partition(date: Date.today)
+          retry
         end
       end
     end

data/lib/messaging/adapters/test/category.rb

@@ -11,10 +11,6 @@ module Messaging
         def messages
           @messages ||= []
         end
-
-        def delete_messages_older_than!(time)
-          messages.delete_if { |m| m.timestamp < time }
-        end
       end
     end
   end

data/lib/messaging/message.rb

@@ -124,6 +124,10 @@ module Messaging
       stream_name.split('$').first
     end
 
+    def category
+      Messaging.category(stream_category)
+    end
+
     def stream_id
       return unless stream_name
 

data/lib/messaging/version.rb

@@ -1,3 +1,3 @@
 module Messaging
-  VERSION = '3.6.2'.freeze
+  VERSION = '3.7.0'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: messaging
 version: !ruby/object:Gem::Version
-  version: 3.6.2
+  version: 3.7.0
 platform: ruby
 authors:
 - Bukowskis
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-02-03 00:00:00.000000000 Z
+date: 2022-02-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -297,7 +297,9 @@ files:
 - lib/messaging/adapters/postgres.rb
 - lib/messaging/adapters/postgres/advisory_transaction_lock.rb
 - lib/messaging/adapters/postgres/categories.rb
+- lib/messaging/adapters/postgres/categories/row.rb
 - lib/messaging/adapters/postgres/category.rb
+- lib/messaging/adapters/postgres/category_with_partitions.rb
 - lib/messaging/adapters/postgres/serialized_message.rb
 - lib/messaging/adapters/postgres/store.rb
 - lib/messaging/adapters/postgres/stream.rb