messaging 3.6.1 → 3.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6a0796bdb3f12989241e5796631d0b2c398801515a4f56cb125f219cc3090102
-  data.tar.gz: f1e8d939623c1e877283889de5d5cef450a121f1b9832cb00ac110964fa64f3b
+  metadata.gz: '049d3a3eac021195a546e67f3ce5351fa93a105a86a7bddc10b6fcb140533e43'
+  data.tar.gz: bce77446512ba9f31d3b21739ef2af2c59ac8bd89c2ff7d15006459aff3cd387
 SHA512:
-  metadata.gz: 6186033688ac0d6c7c713259e92038b35bf405c10258eea01c4b4d17508603c6e2b1a421dc51b1998cfcdc926314121dac9c3cb84337d8091cc456adc097c9fc
-  data.tar.gz: db64b2338a964ab8a52562c837a1afcc9b7b8789ba919fec0f08c5e0ea490b6dae45cd4cbe30fe3ba105c4652b894985ca5b68d50b996cf8a2f79a8bdb95c95c
+  metadata.gz: 2b54e3f6633fa243b6fba4d20e2359c8018cbb37634a7420c75f9e2e8cf6a9da44aa0bebc139ed06ded0b887a35383be72c38c8a58ee6f3e6d288d5527b73e28
+  data.tar.gz: f45a5fb8cee52d26d0ef769a5557ac2fcabebc0b27bc5374a2bd5b25c930a3731b80238efc22429290bfdb0be14b915e2fc995af299d2580b1a6f2160462512f

data/.circleci/config.yml

@@ -5,32 +5,42 @@
 version: 2
 jobs:
   build:
-    machine:
-      image: ubuntu-2004:202111-02
-      docker_layer_caching: true
+    docker:
+      - image: circleci/ruby:2.4.1-node-browsers
+        environment:
+          - CC_TEST_REPORTER_ID: 94ada9b95ee3f232a6e984809d37917cfee90ac47805429e8c49742b2e8d2276
+          - RAILS_ENV: test
+      - image: circleci/postgres:12.5-ram
+        environment:
+          - POSTGRES_HOST_AUTH_METHOD: trust
+          - POSTGRES_PASSWORD: password
+
     working_directory: ~/repo
-    environment:
-      RUBY_VERSION: 2.6.6-buster
-      RAILS_ENV: test
 
     steps:
+      - run:
+          name: Install aptitude because it has much more helpful error messages when ending up in dependency hell
+          command: sudo apt-get install --assume-yes aptitude
+
+      - run:
+          name: Install Postgres Client
+          command: sudo aptitude -y install postgresql-client
+
       - checkout
 
+      - run:
+          name: Setup Code Climate test-reporter
+          command: |
+            curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
+            chmod +x ./cc-test-reporter
+
       - restore_cache:
           name: Restore Bundler Cache
           key: messaging-bundle-{{ checksum "Gemfile.lock" }}
 
-      - run:
-          name: start postgres runner
-          command: docker-compose up -d postgres
-
-      - run:
-          name: Build runner
-          command: docker-compose --verbose build runner
-
       - run:
           name: install dependencies
-          command: bin/de bundle install --jobs=4 --retry=3 --path vendor/bundle
+          command: bundle install --jobs=4 --retry=3 --path vendor/bundle
 
       - save_cache:
           key: messagin-bundle-{{ checksum "Gemfile.lock" }}
@@ -39,11 +49,17 @@ jobs:
 
       - run:
           name: Database Setup
-          command: bin/de bin/setup_db
+          command: |
+            cd spec/example_app
+            bin/rails db:create
+            bin/rails db:migrate
 
       - run:
           name: run tests
-          command: bin/de bundle exec rspec
+          command: |
+            ./cc-test-reporter before-build
+            bundle exec rspec
+            ./cc-test-reporter after-build --exit-code $?
 
 workflows:
   version: 2

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    messaging (3.6.0)
+    messaging (3.5.7)
       activerecord
       activesupport
       after_transaction
@@ -45,8 +45,8 @@ GEM
       tzinfo (~> 1.1)
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
-    after_transaction (0.0.5)
-      activerecord
+    after_transaction (0.0.3)
+      activerecord (< 6.0)
     arel (9.0.0)
     axiom-types (0.1.1)
       descendants_tracker (~> 0.0.4)
@@ -73,23 +73,24 @@ GEM
     descendants_tracker (0.0.4)
       thread_safe (~> 0.3, >= 0.3.1)
     diff-lcs (1.3)
-    digest-crc (0.6.0)
+    digest-crc (0.5.1)
     dnsruby (1.61.3)
       addressable (~> 2.5)
     docile (1.3.2)
-    dry-configurable (0.13.0)
+    dry-configurable (0.8.3)
       concurrent-ruby (~> 1.0)
-      dry-core (~> 0.6)
-    dry-container (0.9.0)
+      dry-core (~> 0.4, >= 0.4.7)
+    dry-container (0.7.2)
       concurrent-ruby (~> 1.0)
-      dry-configurable (~> 0.13, >= 0.13.0)
-    dry-core (0.7.1)
+      dry-configurable (~> 0.1, >= 0.1.3)
+    dry-core (0.4.9)
       concurrent-ruby (~> 1.0)
-    dry-equalizer (0.3.0)
-    dry-initializer (3.0.4)
+    dry-equalizer (0.2.2)
+    dry-initializer (3.0.3)
     em-websocket (0.5.1)
       eventmachine (>= 0.12.9)
       http_parser.rb (~> 0.6.0)
+    equalizer (0.0.11)
     erubi (1.9.0)
     ethon (0.12.0)
       ffi (>= 1.3.0)
@@ -150,7 +151,7 @@ GEM
       octokit (~> 4.0)
       public_suffix (~> 3.0)
       typhoeus (~> 1.3)
-    hashie (5.0.0)
+    hashie (4.1.0)
     html-pipeline (2.12.2)
       activesupport (>= 2)
       nokogiri (>= 1.4)
@@ -284,7 +285,7 @@ GEM
       nokogiri (>= 1.5.9)
     maxminddb (0.1.8)
     mercenary (0.3.6)
-    meter (1.3.0)
+    meter (1.2.7)
       locality (~> 1.1.0)
       useragent (~> 0.16.3)
     method_object (1.0.0)
@@ -357,7 +358,7 @@ GEM
     rspec-support (3.9.0)
     ruby-enum (0.7.2)
       i18n
-    ruby-kafka (1.4.0)
+    ruby-kafka (1.0.0)
       digest-crc
     rubyzip (1.3.0)
     safe_yaml (1.0.5)
@@ -390,10 +391,11 @@ GEM
       thread_safe (~> 0.1)
     unicode-display_width (1.6.0)
     useragent (0.16.10)
-    virtus (2.0.0)
+    virtus (1.0.5)
       axiom-types (~> 0.1)
       coercible (~> 1.0)
       descendants_tracker (~> 0.0, >= 0.0.3)
+      equalizer (~> 0.0, >= 0.0.9)
 
 PLATFORMS
   ruby

data/README.md

@@ -1,36 +1,3 @@
 A library for decoupling applications by using messaging to communicate between components.
 
-## Local development
-
-### Setting up Postgresql and Kafka:
-
-```
-$ docker-compose up -d kafka postgres
-```
-
-If you want to try different version of PG (the default is 12) you can use the environment variable PG_VERSION.
-
-```
-$ PG_VERSION=13 docker-compose up -Vd postgres
-```
-
-The -V option is for recreating the volume instead of using the one from the previous container. The volume are prefixed with the pg version so that they are indenpendent from each other.
-
-
-If you already have PG and Kafka running with another compose project you can skip running them and use the environment variables EXTERNAL_NETWORK and NETWORK_NAME:
-
-```
-$ EXTERNAL_NETWORK=myproject_default NETWORK_NAME=external docker-compose start runner
-```
-
-### Setting up the development environment
-
-The "runner" service is used for interacting with the code. Bundling, starting a console and running specs etc.
-
-After PG and Kafka are up and running just run
-
-```
-$ bin/setup
-```
-
-TODO: Actually implement the bin/setup command :-)
+More documentation and the code will be released in the near future...

data/bin/setup

@@ -1,21 +1,8 @@
-#!/usr/bin/env ruby
-require "fileutils"
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
 
-# path to your application root.
-APP_ROOT = File.expand_path('..', __dir__)
+bundle install
 
-def system!(*args)
-  system(*args) || abort("\n== Command #{args} failed ==")
-end
-
-FileUtils.chdir APP_ROOT do
-  # This script is a way to set up or update your development environment automatically.
-  # This script is idempotent, so that you can run it at any time and get an expectable outcome.
-  # Add necessary setup steps to this file.
-
-  puts "\n== Setup development database =="
-  system! './bin/de rails db:reset'
-
-  puts "\n== Setup test database =="
-  system! './bin/de rails db:reset RAILS_ENV=test'
-end
+# Do any other automated setup that you need to do here

data/lib/messaging/adapters/postgres/categories.rb

@@ -5,21 +5,69 @@ module Messaging
         include Enumerable
 
         def each
-          return enum_for(:each) unless block_given?
-
           all_categories.each do |name|
             yield Category.new(name)
           end
         end
 
+        # Get a category by name
+        #
+        # @param name [String] the name of the category
+        # @return [nil] if no category exists with the given name
+        # @return [Category]
         def [](name)
+          category = all_categories.select { |c| c == name }
+          return unless category
+
+          Category.new(category)
+        end
+
+        # Creates a table partition for the given category
+        #
+        # @param name [String] the name of the category
+        # @return [Category]
+        def create(name)
+          table_name = name.parameterize(separator: '_')
+          sql = <<~SQL
+            CREATE TABLE messaging.#{table_name}
+            PARTITION OF messaging.messages FOR VALUES IN ('#{name}');
+          SQL
+          connection.execute sql
           Category.new(name)
         end
 
+        # Drops the table partition (including all messages) for the given category
+        #
+        # @param name [String] the name of the category
+        def drop(name)
+          table_name = name.parameterize(separator: '_')
+          sql = <<~SQL
+            drop TABLE messaging.#{table_name}
+          SQL
+          connection.execute sql
+        end
+
         private
 
         def all_categories
-          SerializedMessage.distinct.pluck(:stream_category).lazy
+          @all_categories ||= fetch_categories
+        end
+
+        def fetch_categories
+          sql = <<~SQL
+            SELECT child.relname AS category
+            FROM pg_inherits
+            JOIN pg_class parent ON pg_inherits.inhparent = parent.oid
+            JOIN pg_class child  ON pg_inherits.inhrelid   = child.oid
+            JOIN pg_namespace ON pg_namespace.oid  = parent.relnamespace
+            WHERE pg_namespace.nspname = 'messaging' AND child.relkind in ('r', 'p')
+            ORDER BY category
+          SQL
+          connection.select_values(sql)
+        end
+
+        def connection
+          SerializedMessage.connection
         end
       end
     end

data/lib/messaging/adapters/postgres/store.rb

@@ -22,16 +22,11 @@ module Messaging
         # @see Streams
         attr_reader :streams
 
-        # @return [Categories] all the stream categories in the store
-        # @see Categories
-        attr_reader :categories
-
         # Should not be used directly. Access the store though
         # Messaging.message_store or Messaging::Adapters::Store[:postgres]
         # @api private
         def initialize
           @streams = Streams.new
-          @categories = Categories.new
         end
 
         # Get a specific stream by name
@@ -41,8 +36,14 @@ module Messaging
           streams[name]
         end
 
+        # @return [Categories] all the stream categories in the store
+        # @see Categories
+        def categories
+          Categories.new
+        end
+
         # Get a specific category by name
-        # @return [Stream]
+        # @return [Category]
         # @see Messaging.category
         def category(name)
           categories[name]

data/lib/messaging/adapters/postgres.rb

@@ -19,16 +19,11 @@ module Messaging
 
       register!
 
-      def create_schema
-        connection.execute 'CREATE SCHEMA IF NOT EXISTS messaging'
-      end
-
-      def create_id_sequence
-        connection.execute 'CREATE SEQUENCE IF NOT EXISTS messaging.messages_id_seq'
-      end
-
       def create_messages_table
         sql = <<~SQL
+          CREATE SCHEMA IF NOT EXISTS messaging;
+          CREATE SEQUENCE IF NOT EXISTS messaging.messages_id_seq;
+
           CREATE TABLE messaging.messages (
             id bigint DEFAULT nextval('messaging.messages_id_seq'::regclass) NOT NULL,
             uuid uuid NOT NULL,
@@ -50,22 +45,7 @@ module Messaging
         connection.execute sql
       end
 
-      def create_category(category_name)
-        table_name = category_name.parameterize(separator: '_')
-        sql = <<~SQL
-          CREATE TABLE messaging.#{table_name}
-          PARTITION OF messaging.messages FOR VALUES IN ('#{category_name}');
-        SQL
-        connection.execute sql
-      end
-
-      def drop_category(category_name)
-        table_name = category_name.parameterize(separator: '_')
-        sql = <<~SQL
-          drop TABLE messaging.#{table_name}
-        SQL
-        connection.execute sql
-      end
+      private
 
       def connection
         ActiveRecord::Base.connection

data/lib/messaging/config.rb

@@ -2,56 +2,56 @@ module Messaging
   class Config
     extend ::Dry::Configurable
 
-    setting :app_name, default: 'messaging', reader: true
+    setting :app_name, 'messaging', reader: true
 
-    setting(:error_handlers, default: [], reader: true) { |value| Array(value) }
+    setting(:error_handlers, [], reader: true) { |value| Array(value) }
 
-    setting :background_job_handler, default: ->(message, handler_name) { handler_name.constantize.call(message) }, reader: true
+    setting :background_job_handler, ->(message, handler_name) { handler_name.constantize.call(message) }, reader: true
 
-    setting :logger, default: Logger.new(STDOUT), reader: true
+    setting :logger, Logger.new(STDOUT), reader: true
 
     setting :consumer, reader: true do
-      setting :adapter, default: :kafka
-      setting(:middlewares, default: []) { |value| Array(value) }
+      setting :adapter, :kafka
+      setting(:middlewares, []) { |value| Array(value) }
     end
 
     setting :dispatcher, reader: true do
-      setting :adapter, default: :kafka
-      setting(:middlewares, default: []) { |value| Array(value) }
+      setting :adapter, :kafka
+      setting(:middlewares, []) { |value| Array(value) }
     end
 
     setting :message_store, reader: true do
-      setting :adapter, default: :postgres
-      setting(:middlewares, default: []) { |value| Array(value) }
+      setting :adapter, :postgres
+      setting(:middlewares, []) { |value| Array(value) }
     end
 
     setting :kafka, reader: true do
-      setting :log_level, default: :warn
-      setting :pause_timeout, default: 10
+      setting :log_level, :warn
+      setting :pause_timeout, 10
 
       setting :client do
-        setting :seed_brokers, default: ['localhost:9092']
+        setting :seed_brokers, ['localhost:9092']
         setting :connect_timeout
         setting :socket_timeout
         setting :ssl_ca_certs_from_system
         setting :sasl_plain_username
         setting :sasl_plain_password
-        setting :ssl_ca_cert, default: ENV['KAFKA_SSL_CA']
-        setting :ssl_client_cert, default: ENV['KAFKA_SSL_CERTIFICATE']
-        setting :ssl_client_cert_key, default: ENV['KAFKA_SSL_KEY']
+        setting :ssl_ca_cert, ENV['KAFKA_SSL_CA']
+        setting :ssl_client_cert, ENV['KAFKA_SSL_CERTIFICATE']
+        setting :ssl_client_cert_key, ENV['KAFKA_SSL_KEY']
       end
 
       setting :consumer do
-        setting :session_timeout, default: 30
-        setting :offset_commit_interval, default: 10
-        setting :offset_commit_threshold, default: 0
-        setting :heartbeat_interval, default: 10
+        setting :session_timeout, 30
+        setting :offset_commit_interval, 10
+        setting :offset_commit_threshold, 0
+        setting :heartbeat_interval, 10
       end
 
       setting :producer do
-        setting :max_queue_size, default: 5_000
-        setting :delivery_threshold, default: 10
-        setting :delivery_interval, default: 0.05
+        setting :max_queue_size, 5_000
+        setting :delivery_threshold, 10
+        setting :delivery_interval, 0.05
       end
     end
 

data/lib/messaging/version.rb

@@ -1,3 +1,3 @@
 module Messaging
-  VERSION = '3.6.1'.freeze
+  VERSION = '3.6.2'.freeze
 end

data/lib/messaging.rb

@@ -67,6 +67,17 @@ module Messaging
     result
   end
 
+  # Access the stream categories in the current message store
+  #
+  # @example Creating a new category
+  #   Messaging.categories.create('customer')
+  #
+  # @return [Messaging::Adapters::Test::Categories] when using the test adapter
+  # @return [Messaging::Adapters::Postgres::Categories] when using the postgres adapter
+  def self.categories
+    message_store.categories
+  end
+
   def self.category(name)
     message_store.category(name)
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: messaging
 version: !ruby/object:Gem::Version
-  version: 3.6.1
+  version: 3.6.2
 platform: ruby
 authors:
 - Bukowskis
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-02-02 00:00:00.000000000 Z
+date: 2022-02-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -271,10 +271,8 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".circleci/config.yml"
-- ".env"
 - ".gitignore"
 - ".rspec"
-- Dockerfile
 - Gemfile
 - Gemfile.lock
 - README.md
@@ -283,10 +281,7 @@ files:
 - _data/navigation.yml
 - _layouts/single.html
 - bin/console
-- bin/de
 - bin/setup
-- bin/setup_db
-- docker-compose.yml
 - docs/adapters.md
 - docs/consuming.md
 - docs/installation.md

data/.env

@@ -1,2 +0,0 @@
-RUBY_VERSION=2.6.6-buster
-PG_VERSION=12

data/Dockerfile

@@ -1,37 +0,0 @@
-# ** Common setup for all stages
-ARG BASE_IMAGE=ruby:2.6.6-buster
-FROM $BASE_IMAGE
-
-ARG PG_VERSION=12
-ENV LANG=C.UTF-8
-
-# See https://askubuntu.com/questions/972516/debian-frontend-environment-variable/972528#972528
-ENV DEBIAN_FRONTEND=noninteractive
-
-RUN apt-get update -q \
-    && apt-get install -yq --no-install-recommends \
-      build-essential \
-      curl \
-      gnupg \
-      less \
-      postgresql-common \
-      vim \
-      wget
-
-# Install Postgresql apt repository so we can install PG
-RUN (yes || true) | /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh
-
-# Install more packages after updated apt-repos
-RUN apt-get update -q \
-    && apt-get install --no-install-recommends -yq \
-      libpq-dev \
-      postgresql-client-$PG_VERSION
-
-# Create a directory for the app code
-RUN mkdir -p /app
-WORKDIR /app
-ENV PATH=/app/bin:$PATH
-
-# Configure bundler
-ENV BUNDLE_JOBS=4
-ENV BUNDLE_RETRY=3

data/bin/de

@@ -1,11 +0,0 @@
-#!/bin/bash
-
-project_name=${PWD##*/}
-
-get_container_id () {
-  docker container ls | grep "${project_name}_runner" | awk '{print $1}'
-}
-
-container_id=$([[ $(get_container_id) ]] || docker-compose up -d runner && get_container_id)
-
-docker exec -it $container_id "${@}"

data/bin/setup_db

@@ -1,3 +0,0 @@
-#!/bin/bash
-
-cd spec/example_app && bin/rails db:create && bin/rails db:migrate

data/docker-compose.yml

@@ -1,68 +0,0 @@
-version: '3.4'
-
-x-shared: &shared
-    build:
-      context: .
-      dockerfile: ./Dockerfile
-      args:
-        RUBY_VERSION: ${RUBY_VERSION}
-        BASE_IMAGE: ruby:${RUBY_VERSION}
-        PG_VERSION: ${PG_VERSION}
-    tmpfs:
-      - /tmp
-    environment:
-      - RAILS_ENV=${RAILS_ENV:-development}
-      - BOOTSNAP_CACHE_DIR=/rails_cache/bootsnap
-    volumes:
-      - .:/app
-      - bundle:/usr/local/bundle
-    networks:
-      - ${NETWORK_NAME:-default}
-
-services:
-  runner:
-    <<: *shared
-    stdin_open: true
-    tty: true
-    command: /bin/sh
-
-  postgres:
-    image: postgres:${PG_VERSION}
-    volumes:
-      - pg_data:/var/lib/postgresql/data
-    ports:
-      - 5432:5432
-    environment:
-      - POSTGRES_HOST_AUTH_METHOD=trust
-
-  zookeeper:
-    image: zookeeper
-    ports:
-      - "2181:2181"
-
-  kafka:
-    image: niks123123/kafka:2.7.0
-    ports:
-      - "9092:9092"
-    depends_on:
-      - zookeeper
-    environment:
-      KAFKA_ADVERTISED_HOST_NAME: kafka
-      KAFKA_ADVERTISED_PORT: 9092
-      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
-
-volumes:
-  bundle:
-    name: bundle_${RUBY_VERSION}
-  pg_data:
-    name: pg_data_${PG_VERSION}
-
-networks:
-  default:
-    name: messaging_default
-  external:
-    name: ${EXTERNAL_NETWORK}
-    external: true