songkick_queue 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8b51761dcb720a861e59cd35370dd964fbb0393e
-  data.tar.gz: fdcf57c99fa12a9e9e3dbfc9a4c83b8a765486ce
+  metadata.gz: c907eaed6febf237e49b8ff56171ecdd09976fe1
+  data.tar.gz: ac2c3f1207df01dd24623fef5de6492c1b8016ff
 SHA512:
-  metadata.gz: 78d96fe086dfe7afc88548bdfcae4237ecc8fccc95d6d28d24e643b21c058cfd3eb99b8bf49c4e20bff89770763257ace02c56dc672c7d8c6c900f3ea519fabc
-  data.tar.gz: feb0f64206f09160396154611dd732c8327b86f146d82afae897719ae90e097cb9634d008999b17dc3985eef57be9596c0190d7bc979a9467c8d169e9dcd7251
+  metadata.gz: 0c1f1c1cd6b38341badd1b4a0065278730ed6caac7193c4d1c35a374076cd0151bad596eb0b00b7abbba312ae67fe5ff6619978b54e176cdee0e372ce8f02702
+  data.tar.gz: 3d21e628d13199e1467e8bb483432ec6abfe31fbfde4cda5858693450ec1faea565fe1848eb83ddfa2376c4f843739cb8ca0842cba285a9254629b65d01671b5

data/CHANGELOG.md

@@ -1,5 +1,35 @@
 # Changelog
 
+## 0.6.0
+### Added
+- Instrumentation (for tracking metrics using statsd or other tools)
+- Message ID and produced at timestamp to consumer logs
+
+### Changed
+- Bunny gem dependency to `~> 2.2`
+- Reset reconnect attempts back to zero if message is successfully published
+
+## 0.5.0
+### Added
+- Configuration options: `max_reconnect_attempts` and `network_recovery_interval`
+- Reconnect handling on closed connections in producer
+
+## 0.4.0
+### Added
+- Configuration option: `heartbeat_interval`
+
+### Changed
+- Configuration values: replaced single AMQP string with individual config options (eg. host, port etc...)
+
+### Removed
+- Configuration option: `queue_namespace` as RabbitMQ native VHOST can be used instead
+
+## 0.3.0
+### Added
+- Message ID to payload
+- Produced at timestamp to payload
+- Improved documentation
+
 ## 0.2.0
 ### Added
 - Configuration option: `queue_namespace` to namespace queue names when

data/README.md

@@ -71,8 +71,6 @@ class TweetConsumer
     TwitterClient.send_tweet(message[:text], message[:user_id])
   rescue TwitterClient::HttpError => e
     logger.warn(e)
-  rescue StandardError => e
-    logger.error(e)
   end
 end
 ```
@@ -125,6 +123,27 @@ consumer.
 SongkickQueue.publish('notifications-service.tweets', { text: 'Hello world', user_id: 57237722 })
 ```
 
+## Instrumentation
+
+Hooks are provided to instrument producing and consuming of messages using [ActiveSupport's Notifications](http://api.rubyonrails.org/classes/ActiveSupport/Notifications.html) API.
+
+You can subscribe to the following events:
+
+```
+consume_message.songkick_queue
+produce_message.songkick_queue
+```
+
+For both events, the payload includes the message id, produced at timestamp and queue name. The `consume_message` event also includes the consumer class.
+
+For example:
+
+```ruby
+ActiveSupport::Notifications.subscribe('consume_message.songkick_queue') do |name, start, finish, id, payload|
+  # Log info to statsd or something similar
+end
+```
+
 ## Tests
 
 See the current build status on Travis CI: https://travis-ci.org/songkick/queue

data/examples/consumer.rb

@@ -10,6 +10,11 @@ SongkickQueue.configure do |config|
   config.logger = Logger.new(STDOUT)
 end
 
+ActiveSupport::Notifications.subscribe('consume_message.songkick_queue') do |*args|
+  event = ActiveSupport::Notifications::Event.new(*args)
+  SongkickQueue.configuration.logger.info "name: #{event.name}, duration: #{event.duration}, payload: #{event.payload}"
+end
+
 class TweetConsumer
   include SongkickQueue::Consumer
 

data/examples/producer.rb

@@ -10,6 +10,11 @@ SongkickQueue.configure do |config|
   config.logger = Logger.new(STDOUT)
 end
 
+ActiveSupport::Notifications.subscribe('produce_message.songkick_queue') do |*args|
+  event = ActiveSupport::Notifications::Event.new(*args)
+  SongkickQueue.configuration.logger.info "name: #{event.name}, duration: #{event.duration}, payload: #{event.payload}"
+end
+
 3.times do
   SongkickQueue.publish('notifications-service.tweets', { text: 'Hello world', user_id: 57237722 })
 end

data/lib/songkick_queue.rb

@@ -2,6 +2,8 @@ require 'json'
 require 'securerandom'
 require 'logger'
 require 'bunny'
+require 'time'
+require 'active_support/notifications'
 
 require 'songkick_queue/version'
 require 'songkick_queue/client'

data/lib/songkick_queue/producer.rb

@@ -30,9 +30,18 @@ module SongkickQueue
 
       message = JSON.generate(message)
 
-      exchange = client
-        .default_exchange
-        .publish(message, routing_key: String(queue_name))
+      exchange = client.default_exchange
+
+      instrumentation_options = {
+        queue_name: String(queue_name),
+        message_id: message_id,
+        produced_at: produced_at,
+      }
+      ActiveSupport::Notifications.instrument('produce_message.songkick_queue', instrumentation_options) do
+        exchange.publish(message, routing_key: String(queue_name))
+      end
+
+      self.reconnect_attempts = 0
 
       logger.info "Published message #{message_id} to '#{queue_name}' at #{produced_at}"
 

data/lib/songkick_queue/version.rb

@@ -1,3 +1,3 @@
 module SongkickQueue
-  VERSION = "0.5.0"
+  VERSION = "0.6.0"
 end

data/lib/songkick_queue/worker.rb

@@ -83,15 +83,24 @@ module SongkickQueue
     def process_message(consumer_class, delivery_info, properties, message)
       message = JSON.parse(message, symbolize_names: true)
 
-      # Handle both old and new format of messages
-      # TODO: Tidy this up once messages always have a payload
-      payload = message.fetch(:payload, message)
+      message_id = message.fetch(:message_id)
+      produced_at = message.fetch(:produced_at)
+      payload = message.fetch(:payload)
 
-      logger.info "Processing message via #{consumer_class}..."
-      set_process_name(consumer_class)
+      logger.info "Processing message #{message_id} via #{consumer_class}, produced at #{produced_at}"
+      set_process_name(consumer_class, message_id)
 
       consumer = consumer_class.new(delivery_info, logger)
-      consumer.process(payload)
+
+      instrumentation_options = {
+        consumer_class: consumer_class.to_s,
+        queue_name: consumer_class.queue_name,
+        message_id: message_id,
+        produced_at: produced_at,
+      }
+      ActiveSupport::Notifications.instrument('consume_message.songkick_queue', instrumentation_options) do
+        consumer.process(payload)
+      end
     rescue Object => exception
       logger.error(exception)
     ensure
@@ -116,14 +125,19 @@ module SongkickQueue
     # @example idle
     #   set_process_name #=> "songkick_queue[idle]"
     # @example consumer running, namespace is removed
-    #   set_process_name(Foo::TweetConsumer) #=> "songkick_queue[TweetConsumer]"
+    #   set_process_name(Foo::TweetConsumer, 'a729bcd8') #=> "songkick_queue[TweetConsumer#a729bcd8]"
     # @param status [String] of the program
-    def set_process_name(status = 'idle')
+    # @param message_id [String] identifying the message currently being consumed
+    def set_process_name(status = 'idle', message_id = nil)
       formatted_status = String(status)
         .split('::')
         .last
 
-      $PROGRAM_NAME = "#{process_name}[#{formatted_status}]"
+      ident = [formatted_status, message_id]
+        .compact
+        .join('#')
+
+      $PROGRAM_NAME = "#{process_name}[#{ident}]"
     end
   end
 end

data/songkick_queue.gemspec

@@ -25,5 +25,6 @@ Gem::Specification.new do |spec|
   # Used by yardoc for processing README.md code snippets
   spec.add_development_dependency "redcarpet"
 
-  spec.add_dependency "bunny", ">= 1.7.0"
+  spec.add_dependency "bunny", "~> 2.2"
+  spec.add_dependency "activesupport", ">= 3.0.0"
 end

data/spec/songkick_queue/worker_spec.rb

@@ -66,36 +66,11 @@ module SongkickQueue
 
     describe "#process_message" do
       it "should instantiate the consumer and call #process" do
-        ::FooConsumer = Class.new
-        worker = Worker.new(:process_name, FooConsumer)
-
-        logger = double(:logger, info: :null)
-        allow(worker).to receive(:logger) { logger }
-
-        channel = double(:channel, ack: :null)
-        allow(worker).to receive(:channel) { channel }
-
-        delivery_info = double(:delivery_info, delivery_tag: 'tag')
-
-        consumer = double(FooConsumer, process: :null)
-
-        expect(FooConsumer).to receive(:new)
-          .with(delivery_info, logger) { consumer }
-
-        expect(consumer).to receive(:process)
-          .with({ example: 'message', value: true})
-
-        worker.send(:process_message, FooConsumer, delivery_info,
-          :properties, '{"example":"message","value":true}')
-
-        expect(logger).to have_received(:info)
-          .with('Processing message via FooConsumer...')
-
-        expect(channel).to have_received(:ack).with('tag', false)
-      end
-
-      it "should handle new message format with nested payload" do
-        ::BarConsumer = Class.new
+        ::BarConsumer = Struct.new(:delivery_info, :logger) do
+          def self.queue_name
+            "bar-queue"
+          end
+        end
         worker = Worker.new(:process_name, BarConsumer)
 
         logger = double(:logger, info: :null)
@@ -119,7 +94,7 @@ module SongkickQueue
           '"payload":{"example":"message","value":true}}')
 
         expect(logger).to have_received(:info)
-          .with('Processing message via BarConsumer...')
+          .with('Processing message 92c583bdc248 via BarConsumer, produced at 2015-03-30T15:41:55Z')
 
         expect(channel).to have_received(:ack).with('tag', false)
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: songkick_queue
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Dan Lucraft
@@ -9,78 +9,92 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-18 00:00:00.000000000 Z
+date: 2015-10-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.2'
 - !ruby/object:Gem::Dependency
   name: yard
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: redcarpet
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: bunny
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.7.0
+        version: '2.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.7.0
+        version: '2.2'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.0
 description: A gem for processing tasks asynchronously, powered by RabbitMQ.
 email:
 - dan.lucraft@songkick.com
@@ -90,10 +104,10 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .rspec
-- .travis.yml
-- .yardopts
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- ".yardopts"
 - CHANGELOG.md
 - Gemfile
 - LICENSE.txt
@@ -126,17 +140,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.6
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: A gem for processing tasks asynchronously, powered by RabbitMQ.