messaging 3.6.0 → 3.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7733bdd43c5aadf3544db1993e9b5e4c48fe3f8b08aa1f22f52c7677f203a4d3
-  data.tar.gz: b72f6b61afec8c0815f52bd81559eb01e0f7c9ac8f00e98b3f9985f84994b0fe
+  metadata.gz: 6a0796bdb3f12989241e5796631d0b2c398801515a4f56cb125f219cc3090102
+  data.tar.gz: f1e8d939623c1e877283889de5d5cef450a121f1b9832cb00ac110964fa64f3b
 SHA512:
-  metadata.gz: 93f6bee9427950094427800eedbf0ee071de5b8e0a6bafc7ed23a45c85112360d7695fb7d0c0b94b085d504aaf007d4fb647e97b374d20436dd3a26ed02848ca
-  data.tar.gz: b3e9858b73d47d38f7935e0232598e202ef4c834e3183111f376783d4b677c7d680e915efa3da041152cc29ab4329b93478c5dd9d0f595c534f6c8733f0798ed
+  metadata.gz: 6186033688ac0d6c7c713259e92038b35bf405c10258eea01c4b4d17508603c6e2b1a421dc51b1998cfcdc926314121dac9c3cb84337d8091cc456adc097c9fc
+  data.tar.gz: db64b2338a964ab8a52562c837a1afcc9b7b8789ba919fec0f08c5e0ea490b6dae45cd4cbe30fe3ba105c4652b894985ca5b68d50b996cf8a2f79a8bdb95c95c

data/.circleci/config.yml

@@ -5,42 +5,32 @@
 version: 2
 jobs:
   build:
-    docker:
-      # specify the version you desire here
-       - image: circleci/ruby:2.4.1-node-browsers
-         environment:
-           CC_TEST_REPORTER_ID: 94ada9b95ee3f232a6e984809d37917cfee90ac47805429e8c49742b2e8d2276
-           RAILS_ENV: test
-       - image: circleci/postgres:9.6.6-alpine
-         environment:
-           POSTGRES_USER: postgres
-
+    machine:
+      image: ubuntu-2004:202111-02
+      docker_layer_caching: true
     working_directory: ~/repo
+    environment:
+      RUBY_VERSION: 2.6.6-buster
+      RAILS_ENV: test
 
     steps:
-      - run:
-          name: Install aptitude because it has much more helpful error messages when ending up in dependency hell
-          command: sudo apt-get install --assume-yes aptitude
-
-      - run:
-          name: Install Postgres Client
-          command: sudo aptitude -y install postgresql-client
-
       - checkout
 
-      - run:
-          name: Setup Code Climate test-reporter
-          command: |
-            curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
-            chmod +x ./cc-test-reporter
-
       - restore_cache:
           name: Restore Bundler Cache
           key: messaging-bundle-{{ checksum "Gemfile.lock" }}
 
+      - run:
+          name: start postgres runner
+          command: docker-compose up -d postgres
+
+      - run:
+          name: Build runner
+          command: docker-compose --verbose build runner
+
       - run:
           name: install dependencies
-          command: bundle install --jobs=4 --retry=3 --path vendor/bundle
+          command: bin/de bundle install --jobs=4 --retry=3 --path vendor/bundle
 
       - save_cache:
           key: messagin-bundle-{{ checksum "Gemfile.lock" }}
@@ -49,17 +39,11 @@ jobs:
 
       - run:
           name: Database Setup
-          command: |
-            cd spec/example_app
-            bin/rails db:create
-            bin/rails db:migrate
+          command: bin/de bin/setup_db
 
       - run:
           name: run tests
-          command: |
-            ./cc-test-reporter before-build
-            bundle exec rspec
-            ./cc-test-reporter after-build --exit-code $?
+          command: bin/de bundle exec rspec
 
 workflows:
   version: 2

data/.env

@@ -0,0 +1,2 @@
+RUBY_VERSION=2.6.6-buster
+PG_VERSION=12

data/Dockerfile

@@ -0,0 +1,37 @@
+# ** Common setup for all stages
+ARG BASE_IMAGE=ruby:2.6.6-buster
+FROM $BASE_IMAGE
+
+ARG PG_VERSION=12
+ENV LANG=C.UTF-8
+
+# See https://askubuntu.com/questions/972516/debian-frontend-environment-variable/972528#972528
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update -q \
+    && apt-get install -yq --no-install-recommends \
+      build-essential \
+      curl \
+      gnupg \
+      less \
+      postgresql-common \
+      vim \
+      wget
+
+# Install Postgresql apt repository so we can install PG
+RUN (yes || true) | /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh
+
+# Install more packages after updated apt-repos
+RUN apt-get update -q \
+    && apt-get install --no-install-recommends -yq \
+      libpq-dev \
+      postgresql-client-$PG_VERSION
+
+# Create a directory for the app code
+RUN mkdir -p /app
+WORKDIR /app
+ENV PATH=/app/bin:$PATH
+
+# Configure bundler
+ENV BUNDLE_JOBS=4
+ENV BUNDLE_RETRY=3

data/Gemfile

@@ -3,14 +3,14 @@ source 'https://rubygems.org'
 git_source(:github) { |path| "https://github.com/#{path}.git" }
 
 gem 'bootsnap', require: false
-#gem 'listen'
+gem 'listen'
 gem 'pg'
 gem 'pry'
 gem 'pry-rails'
 gem 'railties'
 gem 'sidekiq'
 
-#gem "github-pages", '203',  group: :jekyll_plugins
-#gem "jekyll-include-cache", group: :jekyll_plugins
+gem "github-pages", '203',  group: :jekyll_plugins
+gem "jekyll-include-cache", group: :jekyll_plugins
 
 gemspec

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    messaging (3.5.7)
+    messaging (3.6.0)
       activerecord
       activesupport
       after_transaction
@@ -45,8 +45,8 @@ GEM
       tzinfo (~> 1.1)
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
-    after_transaction (0.0.3)
-      activerecord (< 6.0)
+    after_transaction (0.0.5)
+      activerecord
     arel (9.0.0)
     axiom-types (0.1.1)
       descendants_tracker (~> 0.0.4)
@@ -73,24 +73,23 @@ GEM
     descendants_tracker (0.0.4)
       thread_safe (~> 0.3, >= 0.3.1)
     diff-lcs (1.3)
-    digest-crc (0.5.1)
+    digest-crc (0.6.0)
     dnsruby (1.61.3)
       addressable (~> 2.5)
     docile (1.3.2)
-    dry-configurable (0.8.3)
+    dry-configurable (0.13.0)
       concurrent-ruby (~> 1.0)
-      dry-core (~> 0.4, >= 0.4.7)
-    dry-container (0.7.2)
+      dry-core (~> 0.6)
+    dry-container (0.9.0)
       concurrent-ruby (~> 1.0)
-      dry-configurable (~> 0.1, >= 0.1.3)
-    dry-core (0.4.9)
+      dry-configurable (~> 0.13, >= 0.13.0)
+    dry-core (0.7.1)
       concurrent-ruby (~> 1.0)
-    dry-equalizer (0.2.2)
-    dry-initializer (3.0.3)
+    dry-equalizer (0.3.0)
+    dry-initializer (3.0.4)
     em-websocket (0.5.1)
       eventmachine (>= 0.12.9)
       http_parser.rb (~> 0.6.0)
-    equalizer (0.0.11)
     erubi (1.9.0)
     ethon (0.12.0)
       ffi (>= 1.3.0)
@@ -151,7 +150,7 @@ GEM
       octokit (~> 4.0)
       public_suffix (~> 3.0)
       typhoeus (~> 1.3)
-    hashie (4.1.0)
+    hashie (5.0.0)
     html-pipeline (2.12.2)
       activesupport (>= 2)
       nokogiri (>= 1.4)
@@ -285,7 +284,7 @@ GEM
       nokogiri (>= 1.5.9)
     maxminddb (0.1.8)
     mercenary (0.3.6)
-    meter (1.2.7)
+    meter (1.3.0)
       locality (~> 1.1.0)
       useragent (~> 0.16.3)
     method_object (1.0.0)
@@ -358,7 +357,7 @@ GEM
     rspec-support (3.9.0)
     ruby-enum (0.7.2)
       i18n
-    ruby-kafka (1.0.0)
+    ruby-kafka (1.4.0)
       digest-crc
     rubyzip (1.3.0)
     safe_yaml (1.0.5)
@@ -391,11 +390,10 @@ GEM
       thread_safe (~> 0.1)
     unicode-display_width (1.6.0)
     useragent (0.16.10)
-    virtus (1.0.5)
+    virtus (2.0.0)
       axiom-types (~> 0.1)
       coercible (~> 1.0)
       descendants_tracker (~> 0.0, >= 0.0.3)
-      equalizer (~> 0.0, >= 0.0.9)
 
 PLATFORMS
   ruby

data/README.md

@@ -1,3 +1,36 @@
 A library for decoupling applications by using messaging to communicate between components.
 
-More documentation and the code will be released in the near future...
+## Local development
+
+### Setting up Postgresql and Kafka:
+
+```
+$ docker-compose up -d kafka postgres
+```
+
+If you want to try different version of PG (the default is 12) you can use the environment variable PG_VERSION.
+
+```
+$ PG_VERSION=13 docker-compose up -Vd postgres
+```
+
+The -V option is for recreating the volume instead of using the one from the previous container. The volume are prefixed with the pg version so that they are indenpendent from each other.
+
+
+If you already have PG and Kafka running with another compose project you can skip running them and use the environment variables EXTERNAL_NETWORK and NETWORK_NAME:
+
+```
+$ EXTERNAL_NETWORK=myproject_default NETWORK_NAME=external docker-compose start runner
+```
+
+### Setting up the development environment
+
+The "runner" service is used for interacting with the code. Bundling, starting a console and running specs etc.
+
+After PG and Kafka are up and running just run
+
+```
+$ bin/setup
+```
+
+TODO: Actually implement the bin/setup command :-)

data/bin/de

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+project_name=${PWD##*/}
+
+get_container_id () {
+  docker container ls | grep "${project_name}_runner" | awk '{print $1}'
+}
+
+container_id=$([[ $(get_container_id) ]] || docker-compose up -d runner && get_container_id)
+
+docker exec -it $container_id "${@}"

data/bin/setup

@@ -1,8 +1,21 @@
-#!/usr/bin/env bash
-set -euo pipefail
-IFS=$'\n\t'
-set -vx
+#!/usr/bin/env ruby
+require "fileutils"
 
-bundle install
+# path to your application root.
+APP_ROOT = File.expand_path('..', __dir__)
 
-# Do any other automated setup that you need to do here
+def system!(*args)
+  system(*args) || abort("\n== Command #{args} failed ==")
+end
+
+FileUtils.chdir APP_ROOT do
+  # This script is a way to set up or update your development environment automatically.
+  # This script is idempotent, so that you can run it at any time and get an expectable outcome.
+  # Add necessary setup steps to this file.
+
+  puts "\n== Setup development database =="
+  system! './bin/de rails db:reset'
+
+  puts "\n== Setup test database =="
+  system! './bin/de rails db:reset RAILS_ENV=test'
+end

data/bin/setup_db

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+cd spec/example_app && bin/rails db:create && bin/rails db:migrate

data/docker-compose.yml

@@ -0,0 +1,68 @@
+version: '3.4'
+
+x-shared: &shared
+    build:
+      context: .
+      dockerfile: ./Dockerfile
+      args:
+        RUBY_VERSION: ${RUBY_VERSION}
+        BASE_IMAGE: ruby:${RUBY_VERSION}
+        PG_VERSION: ${PG_VERSION}
+    tmpfs:
+      - /tmp
+    environment:
+      - RAILS_ENV=${RAILS_ENV:-development}
+      - BOOTSNAP_CACHE_DIR=/rails_cache/bootsnap
+    volumes:
+      - .:/app
+      - bundle:/usr/local/bundle
+    networks:
+      - ${NETWORK_NAME:-default}
+
+services:
+  runner:
+    <<: *shared
+    stdin_open: true
+    tty: true
+    command: /bin/sh
+
+  postgres:
+    image: postgres:${PG_VERSION}
+    volumes:
+      - pg_data:/var/lib/postgresql/data
+    ports:
+      - 5432:5432
+    environment:
+      - POSTGRES_HOST_AUTH_METHOD=trust
+
+  zookeeper:
+    image: zookeeper
+    ports:
+      - "2181:2181"
+
+  kafka:
+    image: niks123123/kafka:2.7.0
+    ports:
+      - "9092:9092"
+    depends_on:
+      - zookeeper
+    environment:
+      KAFKA_ADVERTISED_HOST_NAME: kafka
+      KAFKA_ADVERTISED_PORT: 9092
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+
+volumes:
+  bundle:
+    name: bundle_${RUBY_VERSION}
+  pg_data:
+    name: pg_data_${PG_VERSION}
+
+networks:
+  default:
+    name: messaging_default
+  external:
+    name: ${EXTERNAL_NETWORK}
+    external: true

data/lib/messaging/adapters/kafka/producer.rb

@@ -44,7 +44,7 @@ module Messaging
         end
 
         def create_producer
-          kafka.client.async_producer(Config.kafka.producer.to_hash)
+          kafka.client.async_producer(Config.kafka.producer.to_h)
         end
       end
     end

data/lib/messaging/adapters/postgres.rb

@@ -18,6 +18,58 @@ module Messaging
       private_class_method :register!
 
       register!
+
+      def create_schema
+        connection.execute 'CREATE SCHEMA IF NOT EXISTS messaging'
+      end
+
+      def create_id_sequence
+        connection.execute 'CREATE SEQUENCE IF NOT EXISTS messaging.messages_id_seq'
+      end
+
+      def create_messages_table
+        sql = <<~SQL
+          CREATE TABLE messaging.messages (
+            id bigint DEFAULT nextval('messaging.messages_id_seq'::regclass) NOT NULL,
+            uuid uuid NOT NULL,
+            stream character varying NOT NULL,
+            stream_position bigint NOT NULL,
+            message_type character varying NOT NULL,
+            data jsonb,
+            created_at timestamp without time zone NOT NULL,
+            updated_at timestamp without time zone NOT NULL,
+            stream_category character varying,
+            stream_id character varying
+          )
+          PARTITION BY LIST (stream_category);
+
+          CREATE INDEX messages_id_idx ON ONLY messaging.messages USING btree (id);
+          CREATE INDEX messages_stream_category_id_idx ON ONLY messaging.messages USING btree (stream_category, id);
+          CREATE INDEX messages_stream_category_stream_id_stream_position_idx ON ONLY messaging.messages USING btree (stream_category, stream_id, stream_position);
+        SQL
+        connection.execute sql
+      end
+
+      def create_category(category_name)
+        table_name = category_name.parameterize(separator: '_')
+        sql = <<~SQL
+          CREATE TABLE messaging.#{table_name}
+          PARTITION OF messaging.messages FOR VALUES IN ('#{category_name}');
+        SQL
+        connection.execute sql
+      end
+
+      def drop_category(category_name)
+        table_name = category_name.parameterize(separator: '_')
+        sql = <<~SQL
+          drop TABLE messaging.#{table_name}
+        SQL
+        connection.execute sql
+      end
+
+      def connection
+        ActiveRecord::Base.connection
+      end
     end
   end
 end

data/lib/messaging/config.rb

@@ -2,56 +2,56 @@ module Messaging
   class Config
     extend ::Dry::Configurable
 
-    setting :app_name, 'messaging', reader: true
+    setting :app_name, default: 'messaging', reader: true
 
-    setting(:error_handlers, [], reader: true) { |value| Array(value) }
+    setting(:error_handlers, default: [], reader: true) { |value| Array(value) }
 
-    setting :background_job_handler, ->(message, handler_name) { handler_name.constantize.call(message) }, reader: true
+    setting :background_job_handler, default: ->(message, handler_name) { handler_name.constantize.call(message) }, reader: true
 
-    setting :logger, Logger.new(STDOUT), reader: true
+    setting :logger, default: Logger.new(STDOUT), reader: true
 
     setting :consumer, reader: true do
-      setting :adapter, :kafka
-      setting(:middlewares, []) { |value| Array(value) }
+      setting :adapter, default: :kafka
+      setting(:middlewares, default: []) { |value| Array(value) }
     end
 
     setting :dispatcher, reader: true do
-      setting :adapter, :kafka
-      setting(:middlewares, []) { |value| Array(value) }
+      setting :adapter, default: :kafka
+      setting(:middlewares, default: []) { |value| Array(value) }
     end
 
     setting :message_store, reader: true do
-      setting :adapter, :postgres
-      setting(:middlewares, []) { |value| Array(value) }
+      setting :adapter, default: :postgres
+      setting(:middlewares, default: []) { |value| Array(value) }
     end
 
     setting :kafka, reader: true do
-      setting :log_level, :warn
-      setting :pause_timeout, 10
+      setting :log_level, default: :warn
+      setting :pause_timeout, default: 10
 
       setting :client do
-        setting :seed_brokers, ['localhost:9092']
+        setting :seed_brokers, default: ['localhost:9092']
         setting :connect_timeout
         setting :socket_timeout
         setting :ssl_ca_certs_from_system
         setting :sasl_plain_username
         setting :sasl_plain_password
-        setting :ssl_ca_cert, ENV['KAFKA_SSL_CA']
-        setting :ssl_client_cert, ENV['KAFKA_SSL_CERTIFICATE']
-        setting :ssl_client_cert_key, ENV['KAFKA_SSL_KEY']
+        setting :ssl_ca_cert, default: ENV['KAFKA_SSL_CA']
+        setting :ssl_client_cert, default: ENV['KAFKA_SSL_CERTIFICATE']
+        setting :ssl_client_cert_key, default: ENV['KAFKA_SSL_KEY']
       end
 
       setting :consumer do
-        setting :session_timeout, 30
-        setting :offset_commit_interval, 10
-        setting :offset_commit_threshold, 0
-        setting :heartbeat_interval, 10
+        setting :session_timeout, default: 30
+        setting :offset_commit_interval, default: 10
+        setting :offset_commit_threshold, default: 0
+        setting :heartbeat_interval, default: 10
       end
 
       setting :producer do
-        setting :max_queue_size, 5_000
-        setting :delivery_threshold, 10
-        setting :delivery_interval, 0.05
+        setting :max_queue_size, default: 5_000
+        setting :delivery_threshold, default: 10
+        setting :delivery_interval, default: 0.05
       end
     end
 

data/lib/messaging/version.rb

@@ -1,3 +1,3 @@
 module Messaging
-  VERSION = '3.6.0'.freeze
+  VERSION = '3.6.1'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: messaging
 version: !ruby/object:Gem::Version
-  version: 3.6.0
+  version: 3.6.1
 platform: ruby
 authors:
 - Bukowskis
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-01-27 00:00:00.000000000 Z
+date: 2022-02-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -271,8 +271,10 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".circleci/config.yml"
+- ".env"
 - ".gitignore"
 - ".rspec"
+- Dockerfile
 - Gemfile
 - Gemfile.lock
 - README.md
@@ -281,7 +283,10 @@ files:
 - _data/navigation.yml
 - _layouts/single.html
 - bin/console
+- bin/de
 - bin/setup
+- bin/setup_db
+- docker-compose.yml
 - docs/adapters.md
 - docs/consuming.md
 - docs/installation.md