messaging 3.6.2 → 3.7.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '049d3a3eac021195a546e67f3ce5351fa93a105a86a7bddc10b6fcb140533e43'
-  data.tar.gz: bce77446512ba9f31d3b21739ef2af2c59ac8bd89c2ff7d15006459aff3cd387
+  metadata.gz: 33403486d5e5dad21b6ed8b8d5c8e99d090ce19d79f5278e0aa5ee43dd43c8d5
+  data.tar.gz: d174552154912fe26401443c8b5e74b202c0faeb054f8ed6affad299843da91d
 SHA512:
-  metadata.gz: 2b54e3f6633fa243b6fba4d20e2359c8018cbb37634a7420c75f9e2e8cf6a9da44aa0bebc139ed06ded0b887a35383be72c38c8a58ee6f3e6d288d5527b73e28
-  data.tar.gz: f45a5fb8cee52d26d0ef769a5557ac2fcabebc0b27bc5374a2bd5b25c930a3731b80238efc22429290bfdb0be14b915e2fc995af299d2580b1a6f2160462512f
+  metadata.gz: 4477a3ec21fd27ca4735741ec690c8a9f9308c14b18b4c2236a04ba3991dec1fe16359896d1058bb5cf36fda9b2d119cdb8f79870237d1a4d4bed05be6561ac7
+  data.tar.gz: b8ec79f61ac86766d586f166053f829e7329f06e57366cfc88586ffd4a9d28d5047958f9fcf35f1c8ba50f489d48404b623d572197e10b7cdc8f654def0cb5f6

data/.circleci/config.yml

@@ -2,51 +2,53 @@
 #
 # Check https://circleci.com/docs/2.0/language-ruby/ for more details
 #
-version: 2
-jobs:
-  build:
-    docker:
-      - image: circleci/ruby:2.4.1-node-browsers
-        environment:
-          - CC_TEST_REPORTER_ID: 94ada9b95ee3f232a6e984809d37917cfee90ac47805429e8c49742b2e8d2276
-          - RAILS_ENV: test
-      - image: circleci/postgres:12.5-ram
-        environment:
-          - POSTGRES_HOST_AUTH_METHOD: trust
-          - POSTGRES_PASSWORD: password
+version: 2.1
+orbs:
+   slack: circleci/slack@4.8.2
 
-    working_directory: ~/repo
+shared: &shared
+  docker:
+    # https://github.com/bukowskis/fineart/wiki/Updating-the-development-docker-image-for-Circle-CI
+    - image: ghcr.io/bukowskis/base-image/development:ruby-2.6.6-v5
+      auth:
+        username: $GHCR_USER
+        password: $GHCR_PASSWORD
+    - image: cimg/postgres:12.11
+      environment:
+        - POSTGRES_HOST_AUTH_METHOD: trust
+  resource_class: large
+  environment:
+    - CC_TEST_REPORTER_ID: 94ada9b95ee3f232a6e984809d37917cfee90ac47805429e8c49742b2e8d2276
+    - DISABLE_SPRING: 1
+    - RACK_ENV: test
 
+commands:
+  bundle_install:
     steps:
-      - run:
-          name: Install aptitude because it has much more helpful error messages when ending up in dependency hell
-          command: sudo apt-get install --assume-yes aptitude
-
-      - run:
-          name: Install Postgres Client
-          command: sudo aptitude -y install postgresql-client
-
-      - checkout
-
-      - run:
-          name: Setup Code Climate test-reporter
-          command: |
-            curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
-            chmod +x ./cc-test-reporter
-
       - restore_cache:
           name: Restore Bundler Cache
-          key: messaging-bundle-{{ checksum "Gemfile.lock" }}
+          key: bukowskis-bundle-{{ .Environment.CACHE_VERSION }}-{{ checksum "Gemfile.lock" }}
 
       - run:
-          name: install dependencies
-          command: bundle install --jobs=4 --retry=3 --path vendor/bundle
+          name: Bundle Install
+          command: bundle install --path=vendor/bundle --jobs=4 --retry=3
 
       - save_cache:
-          key: messagin-bundle-{{ checksum "Gemfile.lock" }}
+          key: bukowskis-bundle-{{ .Environment.CACHE_VERSION }}-{{ checksum "Gemfile.lock" }}
           paths:
             - vendor/bundle
 
+jobs:
+  specs:
+    <<: *shared
+    steps:
+      - checkout
+      - bundle_install
+      - run:
+          name: Setup Code Climate test-reporter
+          command: |
+            curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
+            chmod +x ./cc-test-reporter
       - run:
           name: Database Setup
           command: |
@@ -65,4 +67,4 @@ workflows:
   version: 2
   messaging_specs:
     jobs:
-      - build
+      - specs

data/.env

@@ -0,0 +1 @@
+PG_VERSION=12

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    messaging (3.5.7)
+    messaging (3.7.1)
       activerecord
       activesupport
       after_transaction
@@ -45,8 +45,8 @@ GEM
       tzinfo (~> 1.1)
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
-    after_transaction (0.0.3)
-      activerecord (< 6.0)
+    after_transaction (0.0.5)
+      activerecord
     arel (9.0.0)
     axiom-types (0.1.1)
       descendants_tracker (~> 0.0.4)
@@ -73,24 +73,23 @@ GEM
     descendants_tracker (0.0.4)
       thread_safe (~> 0.3, >= 0.3.1)
     diff-lcs (1.3)
-    digest-crc (0.5.1)
+    digest-crc (0.6.0)
     dnsruby (1.61.3)
       addressable (~> 2.5)
     docile (1.3.2)
-    dry-configurable (0.8.3)
+    dry-configurable (0.13.0)
       concurrent-ruby (~> 1.0)
-      dry-core (~> 0.4, >= 0.4.7)
-    dry-container (0.7.2)
+      dry-core (~> 0.6)
+    dry-container (0.9.0)
       concurrent-ruby (~> 1.0)
-      dry-configurable (~> 0.1, >= 0.1.3)
-    dry-core (0.4.9)
+      dry-configurable (~> 0.13, >= 0.13.0)
+    dry-core (0.7.1)
       concurrent-ruby (~> 1.0)
-    dry-equalizer (0.2.2)
-    dry-initializer (3.0.3)
+    dry-equalizer (0.3.0)
+    dry-initializer (3.0.4)
     em-websocket (0.5.1)
       eventmachine (>= 0.12.9)
       http_parser.rb (~> 0.6.0)
-    equalizer (0.0.11)
     erubi (1.9.0)
     ethon (0.12.0)
       ffi (>= 1.3.0)
@@ -101,7 +100,6 @@ GEM
     ffi (1.11.3)
     forwardable-extended (2.6.0)
     gemoji (3.0.1)
-    geocoder (1.2.13)
     github-pages (203)
       github-pages-health-check (= 1.16.1)
       jekyll (= 3.8.5)
@@ -151,7 +149,6 @@ GEM
       octokit (~> 4.0)
       public_suffix (~> 3.0)
       typhoeus (~> 1.3)
-    hashie (4.1.0)
     html-pipeline (2.12.2)
       activesupport (>= 2)
       nokogiri (>= 1.4)
@@ -272,21 +269,11 @@ GEM
     listen (3.2.1)
       rb-fsevent (~> 0.10, >= 0.10.3)
       rb-inotify (~> 0.9, >= 0.9.10)
-    locality (1.1.0)
-      activesupport
-      geocoder (= 1.2.13)
-      hashie
-      i18n
-      maxminddb (= 0.1.8)
-      operation
-      trouble
     loofah (2.4.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
-    maxminddb (0.1.8)
     mercenary (0.3.6)
-    meter (1.2.7)
-      locality (~> 1.1.0)
+    meter (1.4.0)
       useragent (~> 0.16.3)
     method_object (1.0.0)
       dry-initializer
@@ -303,7 +290,6 @@ GEM
       mini_portile2 (~> 2.4.0)
     octokit (4.14.0)
       sawyer (~> 0.8.0, >= 0.5.3)
-    operation (1.4.1)
     pathutil (0.16.2)
       forwardable-extended (~> 2.6)
     pg (1.1.4)
@@ -358,7 +344,7 @@ GEM
     rspec-support (3.9.0)
     ruby-enum (0.7.2)
       i18n
-    ruby-kafka (1.0.0)
+    ruby-kafka (1.5.0)
       digest-crc
     rubyzip (1.3.0)
     safe_yaml (1.0.5)
@@ -384,18 +370,16 @@ GEM
       unicode-display_width (~> 1.1, >= 1.1.1)
     thor (0.20.3)
     thread_safe (0.3.6)
-    trouble (0.0.13)
     typhoeus (1.3.1)
       ethon (>= 0.9.0)
     tzinfo (1.2.5)
       thread_safe (~> 0.1)
     unicode-display_width (1.6.0)
     useragent (0.16.10)
-    virtus (1.0.5)
+    virtus (2.0.0)
       axiom-types (~> 0.1)
       coercible (~> 1.0)
       descendants_tracker (~> 0.0, >= 0.0.3)
-      equalizer (~> 0.0, >= 0.0.9)
 
 PLATFORMS
   ruby

data/docker-compose.yml

@@ -0,0 +1,62 @@
+version: '3.4'
+
+x-shared: &shared
+    tmpfs:
+      - /tmp
+    environment:
+      - RAILS_ENV=${RAILS_ENV:-development}
+      - BOOTSNAP_CACHE_DIR=/rails_cache/bootsnap
+    volumes:
+      - .:/app
+      - bundle:/usr/local/bundle
+    networks:
+      - ${NETWORK_NAME:-default}
+
+services:
+  runner:
+    <<: *shared
+    image: ghcr.io/bukowskis/base-image/development:ruby-2.6.6-v5
+    stdin_open: true
+    tty: true
+    command: /bin/sh
+
+  postgres:
+    image: postgres:${PG_VERSION}
+    volumes:
+      - pg_data:/var/lib/postgresql/data
+    ports:
+      - 5432:5432
+    environment:
+      - POSTGRES_HOST_AUTH_METHOD=trust
+
+  zookeeper:
+    image: zookeeper
+    ports:
+      - "2181:2181"
+
+  kafka:
+    image: niks123123/kafka:2.7.0
+    ports:
+      - "9092:9092"
+    depends_on:
+      - zookeeper
+    environment:
+      KAFKA_ADVERTISED_HOST_NAME: kafka
+      KAFKA_ADVERTISED_PORT: 9092
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+
+volumes:
+  bundle:
+    name: bundle
+  pg_data:
+    name: pg_data_${PG_VERSION}
+
+networks:
+  default:
+    name: messaging_default
+  external:
+    name: ${EXTERNAL_NETWORK}
+    external: true

data/lib/messaging/adapters/kafka/producer.rb

@@ -20,8 +20,8 @@ module Messaging
         def call(message)
           reconnect if forked?
           producer.produce(message.to_json, key: message.message_key, topic: message.topic)
-        rescue ::Kafka::BufferOverflow => exception
-          Trouble.notify(exception, pid: Process.pid, message: message.to_json)
+        rescue ::Kafka::BufferOverflow => e
+          ExceptionHandler.call(e, message: message.to_json)
         end
 
         def shutdown

data/lib/messaging/adapters/postgres/categories/row.rb

@@ -0,0 +1,30 @@
+module Messaging
+  module Adapters
+    class Postgres
+      class Categories
+        class Row
+          extend Dry::Initializer
+
+          param :table_name
+          param :type
+          param :expression
+
+          def category_class
+            return CategoryWithPartitions if type == 'p'
+
+            Category
+          end
+
+          def category_name
+            regexp = /FOR VALUES IN \(\'(.*)\'\)/
+            expression.match(regexp)[1]
+          end
+
+          def to_category
+            category_class.new(category_name, table_name)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/messaging/adapters/postgres/categories.rb

@@ -5,9 +5,7 @@ module Messaging
         include Enumerable
 
         def each
-          all_categories.each do |name|
-            yield Category.new(name)
-          end
+          all_categories.each { |c| yield c }
         end
 
         # Get a category by name
@@ -16,10 +14,7 @@ module Messaging
         # @return [nil] if no category exists with the given name
         # @return [Category]
         def [](name)
-          category = all_categories.select { |c| c == name }
-          return unless category
-
-          Category.new(category)
+          all_categories.find { |c| c.name == name }
         end
 
         # Creates a table partition for the given category
@@ -27,20 +22,36 @@ module Messaging
         # @param name [String] the name of the category
         # @return [Category]
         def create(name)
-          table_name = name.parameterize(separator: '_')
+          table_name = Category.table_name_for(name)
           sql = <<~SQL
             CREATE TABLE messaging.#{table_name}
             PARTITION OF messaging.messages FOR VALUES IN ('#{name}');
           SQL
           connection.execute sql
-          Category.new(name)
+          Category.new(name, table_name)
+        end
+
+        # Creates a table partition for the given category
+        # that in turn is partitioned based on created_at
+        #
+        # @param name [String] the name of the category
+        # @return [CategoryWithPartitions]
+        def create_and_partition_by_day(name)
+          table_name = Category.table_name_for(name)
+          sql = <<~SQL
+            CREATE TABLE messaging.#{table_name}
+            PARTITION OF messaging.messages FOR VALUES IN ('#{name}')
+            PARTITION BY RANGE (created_at);
+          SQL
+          connection.execute sql
+          CategoryWithPartitions.new(name, table_name)
         end
 
         # Drops the table partition (including all messages) for the given category
         #
         # @param name [String] the name of the category
         def drop(name)
-          table_name = name.parameterize(separator: '_')
+          table_name = Category.table_name_for(name)
           sql = <<~SQL
             drop TABLE messaging.#{table_name}
           SQL
@@ -55,15 +66,19 @@ module Messaging
 
         def fetch_categories
           sql = <<~SQL
-            SELECT child.relname AS category
+            SELECT child.relname AS category,
+                   child.relkind AS category_type,
+                   pg_get_expr(child.relpartbound, child.oid, true) AS expression
             FROM pg_inherits
             JOIN pg_class parent ON pg_inherits.inhparent = parent.oid
             JOIN pg_class child  ON pg_inherits.inhrelid   = child.oid
             JOIN pg_namespace ON pg_namespace.oid  = parent.relnamespace
-            WHERE pg_namespace.nspname = 'messaging' AND child.relkind in ('r', 'p')
+            WHERE pg_namespace.nspname = 'messaging'
+            AND parent.relname = 'messages'
+            AND child.relkind in ('r', 'p')
             ORDER BY category
           SQL
-          connection.select_values(sql)
+          connection.select_rows(sql).map { |r| Row.new(*r).to_category }
         end
 
         def connection

data/lib/messaging/adapters/postgres/category.rb

@@ -2,16 +2,13 @@ module Messaging
   module Adapters
     class Postgres
       class Category
-        # @return [String] the name of the category
-        attr_reader :name
+        extend Dry::Initializer
 
-        # Should not be used directly.
-        # Use {Messaging.category} or {Store#category}
-        # @api private
-        # @see Messaging.category
-        # @see Store.category
-        def initialize(name)
-          @name = name
+        param :name
+        param :table_name, default: -> { self.class.table_name_for(name) }
+
+        def self.table_name_for(name)
+          name.parameterize(separator: '_')
         end
 
         # Access to all messages in the category sorted by created_at
@@ -20,19 +17,8 @@ module Messaging
           SerializedMessage.where(stream_category: name).order(:created_at)
         end
 
-        def messages_older_than(time)
-          messages.where('created_at < ?', time)
-        end
-
-        def delete_messages_older_than!(time)
-          SerializedMessage.transaction do
-            ActiveRecord::Base.connection.execute "SET LOCAL statement_timeout = '0'"
-            messages_older_than(time).delete_all
-          end
-        end
-
         def inspect
-          "#<Category:#{name}>>"
+          "#<Category: #{name}>"
         end
       end
     end

data/lib/messaging/adapters/postgres/category_with_partitions.rb

@@ -0,0 +1,89 @@
+module Messaging
+  module Adapters
+    class Postgres
+      class CategoryWithPartitions < Category
+        def add_partition(date:)
+          from, to = partition_range_for(date: date)
+          partition_name = partition_name_for(date: from)
+
+          return if partition_exists?(partition_name)
+
+          sql = <<~SQL
+            CREATE TABLE IF NOT EXISTS messaging.#{partition_name}
+            PARTITION OF messaging.#{table_name} FOR VALUES FROM ('#{from}') TO ('#{to}')
+          SQL
+
+          SerializedMessage.transaction do
+            AdvisoryTransactionLock.call key: partition_name
+            SerializedMessage.connection.execute sql
+          end
+        end
+
+        # Creates multiple partitions
+        #
+        # @param start_date [Date] from which date to create partitions
+        # @param days [Integer] how many days worth of partitions to create
+        def add_partitions(start_date:, days: 1)
+          first = start_date.to_date
+          last = first + (days - 1)
+
+          (first..last).each do |date|
+            add_partition(date: date)
+          end
+        end
+
+        # Removes a partition including the included messages
+        #
+        # @param partition_name [String] the name of the partition to drop
+        def drop_partition(partition_name)
+          return unless partition_name.match?(/^#{table_name}_\d{4}_\d{2}_\d{2}$/)
+
+          SerializedMessage.connection.execute "drop TABLE messaging.#{partition_name}"
+        end
+
+        # Removes all partitions older than the given date
+        #
+        # @param date [Date] the cutoff date
+        def drop_partitions_older_than(date)
+          max_partition_name = partition_name_for(date: date)
+          partitions.select { |p| p < max_partition_name }.each do |p|
+            drop_partition(p)
+          end
+        end
+
+        def partition_name_for(date:)
+          "#{name}_%d_%02d_%02d" % [date.year, date.month, date.day]
+        end
+
+        def partition_range_for(date:)
+          from = date.to_time.beginning_of_day
+          to = from + 1.day
+          [from, to]
+        end
+
+        def partition_exists?(partition_name)
+          partitions.include? partition_name
+        end
+
+        def partitions
+          sql = <<~SQL
+            SELECT child.relname AS name
+            FROM pg_inherits
+            JOIN pg_class parent ON pg_inherits.inhparent = parent.oid
+            JOIN pg_class child  ON pg_inherits.inhrelid   = child.oid
+            JOIN pg_namespace ON pg_namespace.oid  = parent.relnamespace
+            WHERE pg_namespace.nspname = 'messaging'
+            AND parent.relname = '#{table_name}'
+            AND child.relkind = 'r'
+            ORDER BY name
+          SQL
+          SerializedMessage.connection.select_values(sql)
+        end
+
+        def inspect
+          "#<CategoryWithPartitions: #{name}>"
+        end
+      end
+    end
+  end
+end

data/lib/messaging/adapters/postgres/store.rb

@@ -1,5 +1,7 @@
 require_relative 'category'
+require_relative 'category_with_partitions'
 require_relative 'categories'
+require_relative 'categories/row'
 require_relative 'stream'
 require_relative 'streams'
 
@@ -83,6 +85,13 @@ module Messaging
           return message unless message.stream_name
 
           SerializedMessage.create!(message: message).to_message
+        rescue ActiveRecord::StatementInvalid => e
+          category = message.category
+          raise e unless e.message.include?('no partition of relation')
+          raise e unless category || category.is_a?(CategoryWithPartitions)
+
+          category.add_partition(date: Date.today)
+          retry
         end
       end
     end

data/lib/messaging/adapters/test/category.rb

@@ -11,10 +11,6 @@ module Messaging
         def messages
           @messages ||= []
         end
-
-        def delete_messages_older_than!(time)
-          messages.delete_if { |m| m.timestamp < time }
-        end
       end
     end
   end

data/lib/messaging/message.rb

@@ -124,6 +124,10 @@ module Messaging
       stream_name.split('$').first
     end
 
+    def category
+      Messaging.category(stream_category)
+    end
+
     def stream_id
       return unless stream_name
 

data/lib/messaging/version.rb

@@ -1,3 +1,3 @@
 module Messaging
-  VERSION = '3.6.2'.freeze
+  VERSION = '3.7.1'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: messaging
 version: !ruby/object:Gem::Version
-  version: 3.6.2
+  version: 3.7.1
 platform: ruby
 authors:
 - Bukowskis
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-02-03 00:00:00.000000000 Z
+date: 2023-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -271,6 +271,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".circleci/config.yml"
+- ".env"
 - ".gitignore"
 - ".rspec"
 - Gemfile
@@ -282,6 +283,7 @@ files:
 - _layouts/single.html
 - bin/console
 - bin/setup
+- docker-compose.yml
 - docs/adapters.md
 - docs/consuming.md
 - docs/installation.md
@@ -297,7 +299,9 @@ files:
 - lib/messaging/adapters/postgres.rb
 - lib/messaging/adapters/postgres/advisory_transaction_lock.rb
 - lib/messaging/adapters/postgres/categories.rb
+- lib/messaging/adapters/postgres/categories/row.rb
 - lib/messaging/adapters/postgres/category.rb
+- lib/messaging/adapters/postgres/category_with_partitions.rb
 - lib/messaging/adapters/postgres/serialized_message.rb
 - lib/messaging/adapters/postgres/store.rb
 - lib/messaging/adapters/postgres/stream.rb