emque-consuming 1.5.0 → 1.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 18f37a4c2c4ca2951d9418cc53d3c4cb3b01ffcb
-  data.tar.gz: cb68592cb698f450c0059caec6ec8b9ccd3b8672
+SHA256:
+  metadata.gz: 4c743d09735f9d7cef0e9afd1c352159e1502abceed3d05fc1132a33597401a5
+  data.tar.gz: c22b17e397c032fc2251393b93feb39558a996258d5c27382b9b3397416a5f61
 SHA512:
-  metadata.gz: b98cc1163c2ea82dc1b73b5e9d05e1851e9baeb7c65ea5c9e84a2f01b7199d3bd80893b22760aeb236d71b71e19df88b53df1bb3e7cce7f615815a74c0b2bc05
-  data.tar.gz: a49054f4f05989287538ea0c1ec25df927e7018641665a9c49c16891106f2fe84113d1723cfeb4dc6493c5fbe17a346db1867114f5d921779a7c46b4c0d86c3d
+  metadata.gz: ea654910256e56c42cf3ef77d88ae0f3f4f57e304b99b54097f36695321dc2a74d2010436ec97b115d1f52895fe25412aeeaa1f49ad8b4e95c0cebf651b730c5
+  data.tar.gz: c5f32b23b28004f8d5ba6f9aafedda6a99ce75b5fe1c136cbea5aea74152b9ef99b01ac8b797eeae5a0f30d60189db39a3e0643251b0a90e2adc76a8a1d54ac5

data/.github/workflows/ci.yml

@@ -0,0 +1,27 @@
+name: CI
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  test:
+
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        ruby: [ '2.3.8', '2.4.10', '2.5.9', '2.6.7', '2.7.3' ]
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+      - name: Install dependencies
+        run: bundle install
+      - name: Run tests
+        run: bundle exec rake

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Emque Consuming CHANGELOG
 
+- [Add support for running in Kubernetes](https://github.com/emque/emque-consuming/pull/84) (1.9.0)
+- [Migrate to GitHub Actions](https://github.com/emque/emque-consuming/pull/82) (1.8.1)
+- [Add prefetch for DelayedMessageWorker](https://github.com/emque/emque-consuming/pull/81) (1.8.0)
+- [Update Rake to fix CVE-2020-8130](https://github.com/emque/emque-consuming/pull/80) (1.7.1)
+- [Update pipe-ruby to remove error handling](https://github.com/emque/emque-consuming/pull/78) 1.7.0
+- [Fixes bug with Bunny 2.12 failing when exchange names are symbols](https://github.com/emque/emque-consuming/pull/77) 1.6.1
+- [Re-enable pipe-ruby `raise_on_error` option to fix automatic retries directing messages to the error queue](https://github.com/emque/emque-consuming/pull/75) 1.6.0
 - [Update the puma gem to allow v3](https://github.com/emque/emque-consuming/pull/72) 1.5.0
 - [Disable pipe-ruby `raise_on_error` option to prevent duplicate erorrs](https://github.com/emque/emque-consuming/pull/74) 1.4.0
 - [Update minimum Ruby version to 2.3](https://github.com/emque/emque-consuming/pull/68) 1.3.0

data/README.md

@@ -1,7 +1,9 @@
-[![Build Status](https://travis-ci.org/emque/emque-consuming.png)](https://travis-ci.org/emque/emque-consuming)
-
 # Emque::Consuming
 
+![CI](https://github.com/emque/emque-consuming/workflows/CI/badge.svg)
+[![Gem Version](https://badge.fury.io/rb/emque-consuming.svg)](https://badge.fury.io/rb/emque-consuming)
+[![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://opensource.org/licenses/MIT)
+
 Emque Consuming is a Ruby application framework that includes everything needed
 to create and run application capable of consuming messages from a message
 broker in a Pub/sub architecture. Messages can be produced with the
@@ -10,7 +12,7 @@ broker in a Pub/sub architecture. Messages can be produced with the
 ## Adapters
 
 We currently only support RabbitMQ. If you would like to add your own adapter,
-take a look at [the adapters directory](https://github.com/emque/emque-consuming/tree/master/lib/emque/consuming/adapters).
+take a look at [the adapters directory](https://github.com/emque/emque-consuming/tree/main/lib/emque/consuming/adapters).
 
 ## Installation
 
@@ -135,7 +137,7 @@ class EventsConsumer
 
   def new_event(message)
     # NOTE: message is an immutable Virtus (https://github.com/solnic/virtus) Value Object.
-    # Check it out here: https://github.com/emque/emque-consuming/blob/master/lib/emque/consuming/message.rb
+    # Check it out here: https://github.com/emque/emque-consuming/blob/main/lib/emque/consuming/message.rb
 
     # You don't have to use pipe (https://github.com/teamsnap/pipe-ruby), but we love it!
     pipe(message, :through => [

data/emque-consuming.gemspec

@@ -6,7 +6,7 @@ require "emque/consuming/version"
 Gem::Specification.new do |spec|
   spec.name          = "emque-consuming"
   spec.version       = Emque::Consuming::VERSION
-  spec.authors       = ["Ryan Williams", "Dan Matthews"]
+  spec.authors       = ["Ryan Williams", "Dan Matthews", "Paul Hanyzewski"]
   spec.email         = ["oss@teamsnap.com"]
   spec.summary       = %q{Microservices framework for Ruby}
   spec.summary       = %q{Microservices framework for Ruby}
@@ -24,11 +24,11 @@ Gem::Specification.new do |spec|
   spec.add_dependency "oj",        "~> 2.18.5"
   spec.add_dependency "virtus",    "~> 1.0"
   spec.add_dependency "puma",      "~> 3.12"
-  spec.add_dependency "pipe-ruby", "~> 0.2.0"
+  spec.add_dependency "pipe-ruby", "~> 1.0"
   spec.add_dependency "inflecto",  "~> 0.0.2"
 
-  spec.add_development_dependency "bundler", "~> 1.7"
-  spec.add_development_dependency "rake",    "~> 10.4.2"
+  spec.add_development_dependency "bundler", ">= 1.17.3"
+  spec.add_development_dependency "rake",    ">= 12.3.3"
   spec.add_development_dependency "rspec",   "~> 3.3"
   spec.add_development_dependency "bunny",   "~> 2.11.0"
   spec.add_development_dependency "timecop", "~> 0.7.1"

data/lib/emque/consuming/adapters/rabbit_mq/delayed_message_worker.rb

@@ -18,6 +18,10 @@ module Emque
           def initialize(connection)
             self.channel = connection.create_channel
 
+            if config.adapter.options[:prefetch]
+              channel.prefetch(config.adapter.options[:prefetch])
+            end
+
             self.delayed_message_exchange = channel.exchange(
               "emque.#{config.app_name}.delayed_message",
               {
@@ -69,8 +73,6 @@ module Emque
               ::Emque::Consuming::Consumer.new.consume(:process, message)
               channel.ack(delivery_info.delivery_tag)
             rescue StandardError => exception
-              logger.error "#{log_prefix} #{exception.class}: #{exception.message}"
-              exception.backtrace.each { |bt| logger.error "#{log_prefix} #{bt}" }
               if retryable_errors.any? { |error| exception.class.to_s =~ /#{error}/ }
                 retry_error(delivery_info, metadata, payload, exception)
               else

data/lib/emque/consuming/adapters/rabbit_mq/error_worker.rb

@@ -15,7 +15,7 @@ module Emque
           def retry_errors
             logger.info "#{log_prefix} starting"
             channel.open if channel.closed?
-            error_queue.message_count.times do
+            [error_queue.message_count, 100].min.times do
               delivery_info, properties, payload = error_queue.pop(
                 {:manual_ack => true}
               )
@@ -56,8 +56,6 @@ module Emque
               ::Emque::Consuming::Consumer.new.consume(:process, message)
               channel.ack(delivery_info.delivery_tag)
             rescue StandardError => exception
-              logger.error "#{log_prefix} #{exception.class}: #{exception.message}"
-              exception.backtrace.each { |bt| logger.error "#{log_prefix} #{bt}" }
               channel.nack(delivery_info.delivery_tag)
             end
           end

data/lib/emque/consuming/adapters/rabbit_mq/worker.rb

@@ -31,19 +31,22 @@ module Emque
               channel.prefetch(config.adapter.options[:prefetch])
             end
 
-            self.queue =
-              channel
-                .queue(
-                  "emque.#{config.app_name}.#{topic}",
-                  :durable => config.adapter.options[:durable],
-                  :auto_delete => config.adapter.options[:auto_delete],
-                  :arguments => {
-                    "x-dead-letter-exchange" => "#{config.app_name}.error"
-                  }
-                )
-                .bind(
-                  channel.fanout(topic, :durable => true, :auto_delete => false)
-                )
+            self.queue = channel
+              .queue(
+                "emque.#{config.app_name}.#{topic}",
+                :durable => config.adapter.options[:durable],
+                :auto_delete => config.adapter.options[:auto_delete],
+                :arguments => {
+                  "x-dead-letter-exchange" => "#{config.app_name}.error"
+                }
+              )
+              .bind(
+                channel.fanout(
+                  topic.to_s, 
+                  :durable => true, 
+                  :auto_delete => false,
+                 )
+              )
           end
 
           def start
@@ -79,8 +82,6 @@ module Emque
               ::Emque::Consuming::Consumer.new.consume(:process, message)
               channel.ack(delivery_info.delivery_tag)
             rescue StandardError => exception
-              logger.error "#{log_prefix} #{exception.class}: #{exception.message}"
-              exception.backtrace.each { |bt| logger.error "#{log_prefix} #{bt}" }
               if enable_delayed_message
                 begin
                   publish_to_delayed_message(delivery_info, metadata, payload)

data/lib/emque/consuming/configuration.rb

@@ -8,7 +8,7 @@ module Emque
         :error_limit, :error_expiration, :retryable_errors,
         :retryable_error_limit, :status, :status_port, :status_host,
         :socket_path, :shutdown_handlers
-      attr_writer :env, :log_level
+      attr_writer :env, :log_level, :log_formatter
 
       def initialize
         @app_name                = ""
@@ -19,6 +19,7 @@ module Emque
         @error_limit             = 5
         @error_expiration        = 3600 # 60 minutes
         @log_level               = nil
+        @log_formatter           = nil
         @retryable_errors        = []
         @retryable_error_limit   = 3
         @status_port             = 10000
@@ -40,6 +41,10 @@ module Emque
         @log_level ||= Logger::INFO
       end
 
+      def log_formatter
+        @log_formatter ||= ::Logger::Formatter.new
+      end
+
       def set_adapter(name, options = {})
         @adapter = Emque::Consuming::Adapter.new(name, options)
       end

data/lib/emque/consuming/consumer.rb

@@ -11,6 +11,9 @@ module Emque
 
       def process(message)
         pipe(message, :through => [:parse, :route])
+      rescue => e
+        handle_error(e, message)
+        raise
       end
 
       private
@@ -29,6 +32,28 @@ module Emque
           message
         )
       end
+
+      def handle_error(e, subject)
+        context = {
+          :consumer => self.class.name,
+          :message => {
+            :current => subject.values,
+            :original => subject.original
+          },
+          :topic => subject.topic
+        }
+
+        # log the error by default
+        Emque::Consuming.logger.error("Error consuming message #{e}")
+        Emque::Consuming.logger.error(context)
+        Emque::Consuming.logger.error e.backtrace.join("\n") unless e.backtrace.nil?
+
+        Emque::Consuming.config.error_handlers.each do |handler|
+          handler.call(e, context)
+        end
+
+        Emque::Consuming.application.instance.notice_error(context)
+      end
     end
   end
 end

data/lib/emque/consuming/consumer/common.rb

@@ -16,7 +16,6 @@ module Emque
         def self.included(descendant)
           descendant.class_eval do
             attr_reader :message
-            private :handle_error, :pipe
           end
         end
 
@@ -26,36 +25,9 @@ module Emque
 
         def pipe_config
           @pipe_config ||= Pipe::Config.new(
-            :error_handlers => [method(:handle_error)],
-            :raise_on_error => false,
             :stop_on => ->(msg, _, _) { !(msg.respond_to?(:continue?) && msg.continue?) }
           )
         end
-
-        def handle_error(e, method:, subject:)
-          context = {
-            :consumer => self.class.name,
-            :message => {
-              :current => subject.values,
-              :original => subject.original
-            },
-            :offset => subject.offset,
-            :partition => subject.partition,
-            :pipe_method => method,
-            :topic => subject.topic
-          }
-
-          # log the error by default
-          Emque::Consuming.logger.error("Error consuming message #{e}")
-          Emque::Consuming.logger.error(context)
-          Emque::Consuming.logger.error e.backtrace.join("\n") unless e.backtrace.nil?
-
-          Emque::Consuming.config.error_handlers.each do |handler|
-            handler.call(e, context)
-          end
-
-          Emque::Consuming.application.instance.notice_error(context)
-        end
       end
     end
   end

data/lib/emque/consuming/core.rb

@@ -50,9 +50,11 @@ module Emque
         )
       end
 
-      def initialize_logger
-        Emque::Consuming::Logging.initialize_logger(logfile)
+      def initialize_logger(daemonized: false)
+        target = daemonized ? logfile : STDOUT
+        Emque::Consuming::Logging.initialize_logger(target)
         Emque::Consuming.logger.level = config.log_level
+        Emque::Consuming.logger.formatter = config.log_formatter
         Celluloid.logger = Emque::Consuming.logger
       end
 
@@ -82,7 +84,7 @@ module Emque
       end
 
       def emque_env
-        config.env_var || ENV["EMQUE_ENV"] || ENV["RACK_ENV"] || "development"
+        config.env_var || ENV["EMQUE_ENV"] || "development"
       end
     end
   end

data/lib/emque/consuming/runner.rb

@@ -23,7 +23,9 @@ module Emque
         self.receivers = []
         self.status = Emque::Consuming::Status.new
         apply_options
-        Emque::Consuming.application.initialize_logger
+        Emque::Consuming
+          .application
+          .initialize_logger(daemonized: options.fetch(:daemon) { false })
         self.class.instance = self
         self.pidfile = options.fetch(:pidfile, default_pidfile)
         self.pid = Emque::Consuming::Pidfile.new(pidfile)

data/lib/emque/consuming/version.rb

@@ -1,5 +1,5 @@
 module Emque
   module Consuming
-    VERSION = "1.5.0"
+    VERSION = "1.9.0"
   end
 end

metadata

@@ -1,15 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: emque-consuming
 version: !ruby/object:Gem::Version
-  version: 1.5.0
+  version: 1.9.0
 platform: ruby
 authors:
 - Ryan Williams
 - Dan Matthews
-autorequire: 
+- Paul Hanyzewski
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-03-04 00:00:00.000000000 Z
+date: 2021-06-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: celluloid
@@ -87,14 +88,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: inflecto
   requirement: !ruby/object:Gem::Requirement
@@ -113,30 +114,30 @@ dependencies:
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: 1.17.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: 1.17.3
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 10.4.2
+        version: 12.3.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 10.4.2
+        version: 12.3.3
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -193,7 +194,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.2.0
-description: 
+description:
 email:
 - oss@teamsnap.com
 executables:
@@ -201,8 +202,8 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/ci.yml"
 - ".gitignore"
-- ".travis.yml"
 - CHANGELOG.md
 - Gemfile
 - LICENSE.txt
@@ -274,7 +275,7 @@ homepage: https://github.com/teamsnap/emque-consuming
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -289,9 +290,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.2.3
-signing_key: 
+rubygems_version: 3.0.3.1
+signing_key:
 specification_version: 4
 summary: Microservices framework for Ruby
 test_files:

data/.travis.yml

@@ -1,11 +0,0 @@
-language: ruby
-rvm:
-  - 2.3.1
-  - 2.3.8
-  - 2.4.5
-  - 2.5.3
-script: bundle exec rake
-notifications:
-  recipients:
-    - shane@teamsnap.com
-    - ryan.williams@teamsnap.com