twingly-amqp 4.2.1 → 5.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 858c1c7b49f3bbd776a0aa0af9f070d1c40f31f9
-  data.tar.gz: ee5ee54beff37137f74fe60d0d4e60dc0db4e893
+SHA256:
+  metadata.gz: c61cf032af90e6b91d1c690ad02f89c6a46e3cc7879813e47f1ee89a1367609a
+  data.tar.gz: 6508f9911ef9dcb8c64854522915129878905e62620dbf022ec104e8a911354a
 SHA512:
-  metadata.gz: ecc06c04e469767a9f9a521989f3ae8f85aec7b18da877f255bda9a12db90575e7d4b5f427624f6d5b7f3e7d94047a6e8ce30c007f14c82f0033341d00c2e12c
-  data.tar.gz: dcf7197d39bdff607daaa44957bb817a4a4f983a697949723051376ac4345595356b0c2b48d25081ae05942948ed8b408df0ee9645e00c7354301ad87af15c18
+  metadata.gz: 0f66b4eda7c713585780fa20f1bae2948b3251f00929f16fd01273f3298a32fb921eaa334e5814f464eac85446bdc17be227e9f850293a0bb753131f061a6f37
+  data.tar.gz: 87c30a0c5e817ea29be19da7b99e3d156f4ba6ca303abd63ebb265827673c90ac3b64ca22bd9d3e5df355163fb29b888931e41d6637282460d598c2bff4114b4

data/README.md

@@ -55,10 +55,11 @@ end
 ```ruby
 subscription = Twingly::AMQP::Subscription.new(
   queue_name:       "crawler-urls",
-  exchange_topic:   "url-exchange", # Optional, uses the default exchange if omitted
-  routing_key:      "url.blog",     # Optional, uses the default exchange if omitted
-  consumer_threads: 4,              # Optional
-  prefetch:         20,             # Optional
+  exchange_topic:   "url-exchange",        # Optional, uses the default exchange if omitted
+  routing_keys:     %w(url.blog url.post), # Optional, uses the default exchange if omitted
+  consumer_threads: 4,                     # Optional
+  prefetch:         20,                    # Optional
+  max_length:       10_000,                # Optional
 )
 
 subscription.on_exception { |exception| puts "Oh noes! #{exception.message}" }
@@ -80,6 +81,19 @@ subscription.each_message do |message| # An instance of Twingly::AMQP::Message
 end
 ```
 
+#### Non-blocking subscription
+
+```ruby
+subscription.each_message(blocking: false) do |message|
+  puts "Received #{message.payload.inspect}"
+  message.ack
+end
+
+# Do something
+
+subscription.cancel! # Stops the subscriber
+```
+
 ### Publish to a queue on the default exchange
 
 ```ruby
@@ -131,9 +145,11 @@ pinger = Twingly::AMQP::Pinger.new(
 
 # Optional, options can also be given to #ping
 pinger.default_ping_options do |options|
-  options.provider_name = "TestProvider"
-  options.source_ip     = "?.?.?.?"
-  options.priority      = 1
+  options.provider_name  = "TestProvider"
+  options.source_ip      = "?.?.?.?"
+  options.priority       = 1
+  # Optional keys/values that will be included in each ping message
+  options.custom_options = { my_option: "This is my own option" }
 end
 
 urls = [
@@ -141,10 +157,12 @@ urls = [
 ]
 
 # Optional, is merged with the default options above
+#   options given to #ping takes precedence over the default options
 options = {
   provider_name: "a-provider-name",
   source_ip:     "?.?.?.?",
   priority:      1,
+  custom_options: { my_other_option: "Another option" },
 }
 
 pinger.ping(urls, options) do |pinged_url|
@@ -197,10 +215,12 @@ bundle exec rake spec:unit
 
 To run static code analysis:
 
-  bundle exec rubocop
+    gem install rubocop
+
+    rubocop
 
-  # optionally on single file(s)
-  bundle exec rubocop lib/twingly/amqp/*
+    # optionally on single file(s)
+    rubocop lib/twingly/amqp/*
 
 ## Release workflow
 

data/lib/twingly/amqp.rb

@@ -4,6 +4,7 @@ require "twingly/amqp/connection"
 require "twingly/amqp/subscription"
 require "twingly/amqp/ping_options"
 require "twingly/amqp/pinger"
+require "twingly/amqp/publisher"
 require "twingly/amqp/default_exchange_publisher"
 require "twingly/amqp/topic_exchange_publisher"
 require "twingly/amqp/null_logger"

data/lib/twingly/amqp/default_exchange_publisher.rb

@@ -1,48 +1,17 @@
+require "twingly/amqp/publisher"
 require "twingly/amqp/connection"
-require "json"
-require "ostruct"
 
 module Twingly
   module AMQP
     class DefaultExchangePublisher
+      include Publisher
+
       def initialize(queue_name:, connection: nil)
         options.routing_key = queue_name
 
         connection ||= Connection.instance
         @exchange = connection.create_channel.default_exchange
       end
-
-      def publish(message)
-        raise ArgumentError unless message.respond_to?(:to_h)
-
-        payload = message.to_h.to_json
-        opts    = options.to_h
-        @exchange.publish(payload, opts)
-      end
-
-      # only used by tests to avoid sleeping
-      def publish_with_confirm(message)
-        channel = @exchange.channel
-        channel.confirm_select unless channel.using_publisher_confirmations?
-
-        publish(message)
-
-        @exchange.wait_for_confirms
-      end
-
-      def configure_publish_options
-        yield options
-      end
-
-      private
-
-      def options
-        @options ||=
-          OpenStruct.new(
-            content_type: "application/json",
-            persistent: true,
-          )
-      end
     end
   end
 end

data/lib/twingly/amqp/ping_options.rb

@@ -3,25 +3,40 @@ module Twingly
     class PingOptions
       attr_accessor :provider_name, :source_ip, :priority
 
-      def initialize(provider_name: nil, source_ip: nil, priority: nil)
-        self.provider_name = provider_name
-        self.source_ip     = source_ip
-        self.priority      = priority
+      attr_reader :custom_options
+
+      def initialize(provider_name: nil, source_ip: nil, priority: nil,
+                     custom_options: {})
+        self.provider_name  = provider_name
+        self.source_ip      = source_ip
+        self.priority       = priority
+        self.custom_options = custom_options
 
         yield self if block_given?
       end
 
+      def custom_options=(options)
+        unless options.respond_to?(:to_h)
+          raise ArgumentError, "custom_options must respond to 'to_h'"
+        end
+
+        @custom_options = options.to_h
+      end
+
       def to_h
         {
-          automatic_ping: false,
           provider_name:  provider_name,
           source_ip:      source_ip,
           priority:       priority,
+          custom_options: custom_options,
         }
       end
 
       def validate
-        missing_keys = to_h.select { |_, value| value.to_s.empty? }.keys
+        missing_keys = to_h.select do |key, value|
+          key != :custom_options && value.to_s.empty?
+        end.keys
+
         if missing_keys.any?
           raise ArgumentError, "Required options not set: #{missing_keys}"
         end
@@ -29,9 +44,10 @@ module Twingly
 
       def merge(other)
         PingOptions.new do |options|
-          options.provider_name = other.provider_name || provider_name
-          options.source_ip     = other.source_ip     || source_ip
-          options.priority      = other.priority      || priority
+          options.provider_name  = other.provider_name || provider_name
+          options.source_ip      = other.source_ip     || source_ip
+          options.priority       = other.priority      || priority
+          options.custom_options = custom_options.merge(other.custom_options)
         end
       end
     end

data/lib/twingly/amqp/pinger.rb

@@ -20,18 +20,17 @@ module Twingly
       end
 
       def ping(urls, options_hash = {})
-        options = PingOptions.new(options_hash)
+        options = PingOptions.new(**options_hash)
         options = @default_ping_options.merge(options)
 
         options.validate
 
         Array(urls).each do |url|
-          unless cached?(url)
-            publish(url, options)
-            cache!(url)
+          next if cached?(url)
+          publish(url, options)
+          cache!(url)
 
-            yield url if block_given?
-          end
+          yield url if block_given?
         end
       end
 
@@ -67,7 +66,7 @@ module Twingly
       end
 
       class NullCache
-        def self.cached?(url)
+        def self.cached?(_url)
           false
         end
 

data/lib/twingly/amqp/publisher.rb

@@ -0,0 +1,47 @@
+require "json"
+require "ostruct"
+
+module Twingly
+  module AMQP
+    module Publisher
+      def publish(message)
+        payload =
+          if message.kind_of?(Array)
+            message
+          elsif message.respond_to?(:to_h)
+            message.to_h
+          else
+            raise ArgumentError
+          end
+
+        json_payload = payload.to_json
+        opts         = options.to_h
+        @exchange.publish(json_payload, opts)
+      end
+
+      # only used by tests to avoid sleeping
+      def publish_with_confirm(message)
+        channel = @exchange.channel
+        channel.confirm_select unless channel.using_publisher_confirmations?
+
+        publish(message)
+
+        @exchange.wait_for_confirms
+      end
+
+      def configure_publish_options
+        yield options
+      end
+
+      private
+
+      def options
+        @options ||=
+          OpenStruct.new(
+            content_type: "application/json",
+            persistent: true,
+          )
+      end
+    end
+  end
+end

data/lib/twingly/amqp/session.rb

@@ -27,10 +27,6 @@ module Twingly
         options[:pass]  ||= password_from_env
         options[:hosts] ||= hosts_from_env
 
-        # Used as a workaround for
-        # https://github.com/ruby-amqp/bunny/issues/446
-        options[:hosts] = options.fetch(:hosts).dup
-
         default_connection_options.merge(options)
       end
 

data/lib/twingly/amqp/subscription.rb

@@ -4,46 +4,45 @@ require "twingly/amqp/message"
 module Twingly
   module AMQP
     class Subscription
-      def initialize(queue_name:, exchange_topic: nil, routing_key: nil, consumer_threads: 4, prefetch: 20, connection: nil)
+      def initialize(queue_name:, exchange_topic: nil, routing_key: nil,
+                     routing_keys: nil, consumer_threads: 1, prefetch: 20,
+                     connection: nil, max_length: nil)
         @queue_name       = queue_name
         @exchange_topic   = exchange_topic
-        @routing_key      = routing_key
+        @routing_keys     = Array(routing_keys || routing_key)
         @consumer_threads = consumer_threads
         @prefetch         = prefetch
+        @max_length       = max_length
+
+        if routing_key
+          warn "[DEPRECATION] `routing_key` is deprecated. "\
+               "Please use `routing_keys` instead."
+        end
 
         connection ||= Connection.instance
         @channel = create_channel(connection)
         @queue   = @channel.queue(@queue_name, queue_options)
 
-        if @exchange_topic && @routing_key
+        if @exchange_topic && @routing_keys.any?
           exchange = @channel.topic(@exchange_topic, durable: true)
-          @queue.bind(exchange, routing_key: @routing_key)
+
+          @routing_keys.each do |routing_key|
+            @queue.bind(exchange, routing_key: routing_key)
+          end
         end
 
         @before_handle_message_callback = proc {}
         @on_exception_callback          = proc {}
       end
 
-      def each_message(&block)
-        setup_traps
+      def each_message(blocking: true, &block)
+        consumer = create_consumer(&block)
 
-        consumer = @queue.subscribe(subscribe_options) do |delivery_info, metadata, payload|
-          @before_handle_message_callback.call(payload)
+        if blocking
+          sleep 0.01 until cancel?
 
-          message = Message.new(
-            delivery_info: delivery_info,
-            metadata:      metadata,
-            payload:       payload,
-            channel:       @channel,
-          )
-
-          block.call(message)
+          consumer.cancel
         end
-
-        # The consumer isn't blocking, so we wait here
-        sleep 0.5 until cancel?
-
-        consumer.cancel
       end
 
       def before_handle_message(&block)
@@ -54,6 +53,14 @@ module Twingly
         @on_exception_callback = block
       end
 
+      def message_count
+        @queue.status.fetch(:message_count)
+      end
+
+      def raw_queue
+        @queue
+      end
+
       def cancel?
         @cancel
       end
@@ -64,6 +71,21 @@ module Twingly
 
       private
 
+      def create_consumer
+        @queue.subscribe(subscribe_options) do |delivery_info, metadata, payload|
+          @before_handle_message_callback.call(payload)
+
+          message = Message.new(
+            delivery_info: delivery_info,
+            metadata:      metadata,
+            payload:       payload,
+            channel:       @channel,
+          )
+
+          yield message
+        end
+      end
+
       def create_channel(connection)
         channel = connection.create_channel(nil, @consumer_threads)
         channel.prefetch(@prefetch)
@@ -77,9 +99,16 @@ module Twingly
       def queue_options
         {
           durable: true,
+          arguments: queue_arguments,
         }
       end
 
+      def queue_arguments
+        {}.tap do |arguments|
+          arguments["x-max-length"] = @max_length if @max_length
+        end
+      end
+
       def subscribe_options
         {
           manual_ack:   true,
@@ -91,18 +120,6 @@ module Twingly
         tag_name = Socket.gethostname
         @channel.generate_consumer_tag(tag_name)
       end
-
-      def setup_traps
-        [:INT, :TERM].each do |signal|
-          Signal.trap(signal) do
-            # Exit fast if we've already got a signal since before
-            exit!(true) if cancel?
-
-            # Set cancel flag, cancels consumers
-            cancel!
-          end
-        end
-      end
     end
   end
 end

data/lib/twingly/amqp/topic_exchange_publisher.rb

@@ -1,48 +1,17 @@
+require "twingly/amqp/publisher"
 require "twingly/amqp/connection"
-require "json"
-require "ostruct"
 
 module Twingly
   module AMQP
     class TopicExchangePublisher
+      include Publisher
+
       def initialize(exchange_name:, routing_key: nil, connection: nil, opts: {})
         options.routing_key = routing_key
 
         connection ||= Connection.instance
         @exchange = connection.create_channel.topic(exchange_name, opts)
       end
-
-      def publish(message)
-        raise ArgumentError unless message.respond_to?(:to_h)
-
-        payload = message.to_h.to_json
-        opts    = options.to_h
-        @exchange.publish(payload, opts)
-      end
-
-      # only used by tests to avoid sleeping
-      def publish_with_confirm(message)
-        channel = @exchange.channel
-        channel.confirm_select unless channel.using_publisher_confirmations?
-
-        publish(message)
-
-        @exchange.wait_for_confirms
-      end
-
-      def configure_publish_options
-        yield options
-      end
-
-      private
-
-      def options
-        @options ||=
-          OpenStruct.new(
-            content_type: "application/json",
-            persistent: true,
-          )
-      end
     end
   end
 end

data/lib/twingly/amqp/version.rb

@@ -1,5 +1,5 @@
 module Twingly
   module Amqp
-    VERSION = "4.2.1".freeze
+    VERSION = "5.0.1".freeze
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: twingly-amqp
 version: !ruby/object:Gem::Version
-  version: 4.2.1
+  version: 5.0.1
 platform: ruby
 authors:
 - Twingly AB
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-11-17 00:00:00.000000000 Z
+date: 2020-09-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bunny
@@ -16,56 +16,48 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.2'
+        version: '2.6'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.6.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.2'
-- !ruby/object:Gem::Dependency
-  name: rspec
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
+        version: '2.6'
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3'
+        version: 2.6.2
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12'
 - !ruby/object:Gem::Dependency
-  name: rubocop
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3'
 description: Publish and subscribe to messages via RabbitMQ
 email:
 - support@twingly.com
@@ -81,6 +73,7 @@ files:
 - lib/twingly/amqp/null_logger.rb
 - lib/twingly/amqp/ping_options.rb
 - lib/twingly/amqp/pinger.rb
+- lib/twingly/amqp/publisher.rb
 - lib/twingly/amqp/session.rb
 - lib/twingly/amqp/subscription.rb
 - lib/twingly/amqp/topic_exchange_publisher.rb
@@ -88,7 +81,7 @@ files:
 homepage: https://github.com/twingly/twingly-amqp
 licenses: []
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -103,9 +96,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.4.5.1
-signing_key: 
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
 summary: Ruby library for talking to RabbitMQ
 test_files: []