messaging 3.5.7 → 3.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 44057f3b83334d34fb1cefa4b0261d84ef93bf764c487ae7084c425dac105643
-  data.tar.gz: 38a69af6df0a7d1b86f492d21d3b4ffe3e0d2e4ff360fe53cc106f499da6879f
+  metadata.gz: 860c436d2faab463d40144358859eb282aa352370ffb2c725cfa292c267eb959
+  data.tar.gz: cf39e2bf8a10ebe84cfe09d9e74d0fa9bce654803478865b47601b5ada04d404
 SHA512:
-  metadata.gz: c1119c1db6d97fe860cfd82dcb9547546b5313caf174557756453a0aa354823a335a306784e35c022d2f0a693057707789c56fa34369e58c81f0555ad457ce4b
-  data.tar.gz: 90640e22a90b03de047daf4a228c81defa88f56321eca71f725ad898c9dae182edab6e7523408bde73d9f1fb26a3ce1efe08db9b298d96500f4f1967c963d461
+  metadata.gz: '0751658ec0a77cc8fd925963c1530c05ee31cb458c8f773a8aba7dd333569e90547945d52dda5cbee21e0f0668cf6d7909cb2f2bd598815a4145cff375233573'
+  data.tar.gz: 0abc8205f0f400f5cade71abe7f075a96338beb44c8559c79b5603f478ddbd8dec7dee4a7ca56784e2be2f10238f16667fd2d68a7e7e54b36caa0529fd62fd95

data/.circleci/config.yml

@@ -6,14 +6,14 @@ version: 2
 jobs:
   build:
     docker:
-      # specify the version you desire here
-       - image: circleci/ruby:2.4.1-node-browsers
-         environment:
-           CC_TEST_REPORTER_ID: 94ada9b95ee3f232a6e984809d37917cfee90ac47805429e8c49742b2e8d2276
-           RAILS_ENV: test
-       - image: circleci/postgres:9.6.6-alpine
-         environment:
-           POSTGRES_USER: postgres
+      - image: circleci/ruby:2.4.1-node-browsers
+        environment:
+          - CC_TEST_REPORTER_ID: 94ada9b95ee3f232a6e984809d37917cfee90ac47805429e8c49742b2e8d2276
+          - RAILS_ENV: test
+      - image: circleci/postgres:12.5-ram
+        environment:
+          - POSTGRES_HOST_AUTH_METHOD: trust
+          - POSTGRES_PASSWORD: password
 
     working_directory: ~/repo
 

data/lib/messaging/adapters/kafka/producer.rb

@@ -44,7 +44,7 @@ module Messaging
         end
 
         def create_producer
-          kafka.client.async_producer(Config.kafka.producer.to_hash)
+          kafka.client.async_producer(Config.kafka.producer.to_h)
         end
       end
     end

data/lib/messaging/adapters/postgres/categories/row.rb

@@ -0,0 +1,30 @@
+module Messaging
+  module Adapters
+    class Postgres
+      class Categories
+        class Row
+          extend Dry::Initializer
+
+          param :table_name
+          param :type
+          param :expression
+
+          def category_class
+            return CategoryWithPartitions if type == 'p'
+
+            Category
+          end
+
+          def category_name
+            regexp = /FOR VALUES IN \(\'(.*)\'\)/
+            expression.match(regexp)[1]
+          end
+
+          def to_category
+            category_class.new(category_name, table_name)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/messaging/adapters/postgres/categories.rb

@@ -5,21 +5,84 @@ module Messaging
         include Enumerable
 
         def each
-          return enum_for(:each) unless block_given?
-
-          all_categories.each do |name|
-            yield Category.new(name)
-          end
+          all_categories.each { |c| yield c }
         end
 
+        # Get a category by name
+        #
+        # @param name [String] the name of the category
+        # @return [nil] if no category exists with the given name
+        # @return [Category]
         def [](name)
-          Category.new(name)
+          all_categories.find { |c| c.name == name }
+        end
+
+        # Creates a table partition for the given category
+        #
+        # @param name [String] the name of the category
+        # @return [Category]
+        def create(name)
+          table_name = Category.table_name_for(name)
+          sql = <<~SQL
+            CREATE TABLE messaging.#{table_name}
+            PARTITION OF messaging.messages FOR VALUES IN ('#{name}');
+          SQL
+          connection.execute sql
+          Category.new(name, table_name)
+        end
+
+        # Creates a table partition for the given category
+        # that in turn is partitioned based on created_at
+        #
+        # @param name [String] the name of the category
+        # @return [CategoryWithPartitions]
+        def create_and_partition_by_day(name)
+          table_name = Category.table_name_for(name)
+          sql = <<~SQL
+            CREATE TABLE messaging.#{table_name}
+            PARTITION OF messaging.messages FOR VALUES IN ('#{name}')
+            PARTITION BY RANGE (created_at);
+          SQL
+          connection.execute sql
+          CategoryWithPartitions.new(name, table_name)
+        end
+
+        # Drops the table partition (including all messages) for the given category
+        #
+        # @param name [String] the name of the category
+        def drop(name)
+          table_name = Category.table_name_for(name)
+          sql = <<~SQL
+            drop TABLE messaging.#{table_name}
+          SQL
+          connection.execute sql
         end
 
         private
 
         def all_categories
-          SerializedMessage.distinct.pluck(:stream_category).lazy
+          @all_categories ||= fetch_categories
+        end
+
+        def fetch_categories
+          sql = <<~SQL
+            SELECT child.relname AS category,
+                   child.relkind AS category_type,
+                   pg_get_expr(child.relpartbound, child.oid, true) AS expression
+            FROM pg_inherits
+            JOIN pg_class parent ON pg_inherits.inhparent = parent.oid
+            JOIN pg_class child  ON pg_inherits.inhrelid   = child.oid
+            JOIN pg_namespace ON pg_namespace.oid  = parent.relnamespace
+            WHERE pg_namespace.nspname = 'messaging'
+            AND parent.relname = 'messages'
+            AND child.relkind in ('r', 'p')
+            ORDER BY category
+          SQL
+          connection.select_rows(sql).map { |r| Row.new(*r).to_category }
+        end
+
+        def connection
+          SerializedMessage.connection
         end
       end
     end

data/lib/messaging/adapters/postgres/category.rb

@@ -2,16 +2,13 @@ module Messaging
   module Adapters
     class Postgres
       class Category
-        # @return [String] the name of the category
-        attr_reader :name
+        extend Dry::Initializer
 
-        # Should not be used directly.
-        # Use {Messaging.category} or {Store#category}
-        # @api private
-        # @see Messaging.category
-        # @see Store.category
-        def initialize(name)
-          @name = name
+        param :name
+        param :table_name, default: -> { self.class.table_name_for(name) }
+
+        def self.table_name_for(name)
+          name.parameterize(separator: '_')
         end
 
         # Access to all messages in the category sorted by created_at
@@ -20,19 +17,8 @@ module Messaging
           SerializedMessage.where(stream_category: name).order(:created_at)
         end
 
-        def messages_older_than(time)
-          messages.where('created_at < ?', time)
-        end
-
-        def delete_messages_older_than!(time)
-          SerializedMessage.transaction do
-            ActiveRecord::Base.connection.execute "SET LOCAL statement_timeout = '0'"
-            messages_older_than(time).delete_all
-          end
-        end
-
         def inspect
-          "#<Category:#{name}>>"
+          "#<Category: #{name}>"
         end
       end
     end

data/lib/messaging/adapters/postgres/category_with_partitions.rb

@@ -0,0 +1,89 @@
+module Messaging
+  module Adapters
+    class Postgres
+      class CategoryWithPartitions < Category
+        def add_partition(date:)
+          from, to = partition_range_for(date: date)
+          partition_name = partition_name_for(date: from)
+
+          return if partition_exists?(partition_name)
+
+          sql = <<~SQL
+            CREATE TABLE IF NOT EXISTS messaging.#{partition_name}
+            PARTITION OF messaging.#{table_name} FOR VALUES FROM ('#{from}') TO ('#{to}')
+          SQL
+
+          SerializedMessage.transaction do
+            AdvisoryTransactionLock.call key: partition_name
+            SerializedMessage.connection.execute sql
+          end
+        end
+
+        # Creates multiple partitions
+        #
+        # @param start_date [Date] from which date to create partitions
+        # @param days [Integer] how many days worth of partitions to create
+        def add_partitions(start_date:, days: 1)
+          first = start_date.to_date
+          last = first + (days - 1)
+
+          (first..last).each do |date|
+            add_partition(date: date)
+          end
+        end
+
+        # Removes a partition including the included messages
+        #
+        # @param partition_name [String] the name of the partition to drop
+        def drop_partition(partition_name)
+          return unless partition_name.match?(/^#{table_name}_\d{4}_\d{2}_\d{2}$/)
+
+          SerializedMessage.connection.execute "drop TABLE messaging.#{partition_name}"
+        end
+
+        # Removes all partitions older than the given date
+        #
+        # @param date [Date] the cutoff date
+        def drop_partitions_older_than(date)
+          max_partition_name = partition_name_for(date: date)
+          partitions.select { |p| p < max_partition_name }.each do |p|
+            drop_partition(p)
+          end
+        end
+
+        def partition_name_for(date:)
+          "#{name}_%d_%02d_%02d" % [date.year, date.month, date.day]
+        end
+
+        def partition_range_for(date:)
+          from = date.to_time.beginning_of_day
+          to = from + 1.day
+          [from, to]
+        end
+
+        def partition_exists?(partition_name)
+          partitions.include? partition_name
+        end
+
+        def partitions
+          sql = <<~SQL
+            SELECT child.relname AS name
+            FROM pg_inherits
+            JOIN pg_class parent ON pg_inherits.inhparent = parent.oid
+            JOIN pg_class child  ON pg_inherits.inhrelid   = child.oid
+            JOIN pg_namespace ON pg_namespace.oid  = parent.relnamespace
+            WHERE pg_namespace.nspname = 'messaging'
+            AND parent.relname = '#{table_name}'
+            AND child.relkind = 'r'
+            ORDER BY name
+          SQL
+          SerializedMessage.connection.select_values(sql)
+        end
+
+        def inspect
+          "#<CategoryWithPartitions: #{name}>"
+        end
+      end
+    end
+  end
+end

data/lib/messaging/adapters/postgres/serialized_message.rb

@@ -2,7 +2,7 @@ module Messaging
   module Adapters
     class Postgres
       class SerializedMessage < ActiveRecord::Base
-        self.table_name = :messaging_messages
+        self.table_name = 'messaging.messages'
 
         attr_accessor :expected_version
 

data/lib/messaging/adapters/postgres/store.rb

@@ -1,5 +1,7 @@
 require_relative 'category'
+require_relative 'category_with_partitions'
 require_relative 'categories'
+require_relative 'categories/row'
 require_relative 'stream'
 require_relative 'streams'
 
@@ -22,16 +24,11 @@ module Messaging
         # @see Streams
         attr_reader :streams
 
-        # @return [Categories] all the stream categories in the store
-        # @see Categories
-        attr_reader :categories
-
         # Should not be used directly. Access the store though
         # Messaging.message_store or Messaging::Adapters::Store[:postgres]
         # @api private
         def initialize
           @streams = Streams.new
-          @categories = Categories.new
         end
 
         # Get a specific stream by name
@@ -41,8 +38,14 @@ module Messaging
           streams[name]
         end
 
+        # @return [Categories] all the stream categories in the store
+        # @see Categories
+        def categories
+          Categories.new
+        end
+
         # Get a specific category by name
-        # @return [Stream]
+        # @return [Category]
         # @see Messaging.category
         def category(name)
           categories[name]
@@ -82,6 +85,13 @@ module Messaging
           return message unless message.stream_name
 
           SerializedMessage.create!(message: message).to_message
+        rescue ActiveRecord::StatementInvalid => e
+          category = message.category
+          raise e unless e.message.include?('no partition of relation')
+          raise e unless category || category.is_a?(CategoryWithPartitions)
+
+          category.add_partition(date: Date.today)
+          retry
         end
       end
     end

data/lib/messaging/adapters/postgres.rb

@@ -18,6 +18,38 @@ module Messaging
       private_class_method :register!
 
       register!
+
+      def create_messages_table
+        sql = <<~SQL
+          CREATE SCHEMA IF NOT EXISTS messaging;
+          CREATE SEQUENCE IF NOT EXISTS messaging.messages_id_seq;
+
+          CREATE TABLE messaging.messages (
+            id bigint DEFAULT nextval('messaging.messages_id_seq'::regclass) NOT NULL,
+            uuid uuid NOT NULL,
+            stream character varying NOT NULL,
+            stream_position bigint NOT NULL,
+            message_type character varying NOT NULL,
+            data jsonb,
+            created_at timestamp without time zone NOT NULL,
+            updated_at timestamp without time zone NOT NULL,
+            stream_category character varying,
+            stream_id character varying
+          )
+          PARTITION BY LIST (stream_category);
+
+          CREATE INDEX messages_id_idx ON ONLY messaging.messages USING btree (id);
+          CREATE INDEX messages_stream_category_id_idx ON ONLY messaging.messages USING btree (stream_category, id);
+          CREATE INDEX messages_stream_category_stream_id_stream_position_idx ON ONLY messaging.messages USING btree (stream_category, stream_id, stream_position);
+        SQL
+        connection.execute sql
+      end
+
+      private
+
+      def connection
+        ActiveRecord::Base.connection
+      end
     end
   end
 end

data/lib/messaging/adapters/test/category.rb

@@ -11,10 +11,6 @@ module Messaging
         def messages
           @messages ||= []
         end
-
-        def delete_messages_older_than!(time)
-          messages.delete_if { |m| m.timestamp < time }
-        end
       end
     end
   end

data/lib/messaging/message.rb

@@ -124,6 +124,10 @@ module Messaging
       stream_name.split('$').first
     end
 
+    def category
+      Messaging.category(stream_category)
+    end
+
     def stream_id
       return unless stream_name
 

data/lib/messaging/version.rb

@@ -1,3 +1,3 @@
 module Messaging
-  VERSION = '3.5.7'.freeze
+  VERSION = '3.7.0'.freeze
 end

data/lib/messaging.rb

@@ -67,6 +67,17 @@ module Messaging
     result
   end
 
+  # Access the stream categories in the current message store
+  #
+  # @example Creating a new category
+  #   Messaging.categories.create('customer')
+  #
+  # @return [Messaging::Adapters::Test::Categories] when using the test adapter
+  # @return [Messaging::Adapters::Postgres::Categories] when using the postgres adapter
+  def self.categories
+    message_store.categories
+  end
+
   def self.category(name)
     message_store.category(name)
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: messaging
 version: !ruby/object:Gem::Version
-  version: 3.5.7
+  version: 3.7.0
 platform: ruby
 authors:
 - Bukowskis
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-05-31 00:00:00.000000000 Z
+date: 2022-02-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -273,7 +273,6 @@ files:
 - ".circleci/config.yml"
 - ".gitignore"
 - ".rspec"
-- ".ruby-version"
 - Gemfile
 - Gemfile.lock
 - README.md
@@ -298,7 +297,9 @@ files:
 - lib/messaging/adapters/postgres.rb
 - lib/messaging/adapters/postgres/advisory_transaction_lock.rb
 - lib/messaging/adapters/postgres/categories.rb
+- lib/messaging/adapters/postgres/categories/row.rb
 - lib/messaging/adapters/postgres/category.rb
+- lib/messaging/adapters/postgres/category_with_partitions.rb
 - lib/messaging/adapters/postgres/serialized_message.rb
 - lib/messaging/adapters/postgres/store.rb
 - lib/messaging/adapters/postgres/stream.rb
@@ -351,7 +352,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.0.3.1
 signing_key: 
 specification_version: 4
 summary: A library for decoupling applications by using messaging to communicate between

data/.ruby-version

@@ -1 +0,0 @@
-2.3.1