fleck 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3844f5f141564257d45c59386c82683d9adc62ec
-  data.tar.gz: d5fec104b21c27ebd65b439a66ac702617324d93
+  metadata.gz: 0a342aa3de500e0ece6c60e76e57122a869c5e74
+  data.tar.gz: 5256c6fcebf654fc57e86189b6c1023b54d6e742
 SHA512:
-  metadata.gz: bd36645f66fea70b5347acbb86673eee29ac8a3e89e02ad43dcc0c9d5d585efbe38f5d3ebb175ae5f4075aa94779387c4e17dfc8839f0e6029642aa8a95b10c7
-  data.tar.gz: e107b55e67021305429092c26c69d62f5c7af55f011763ab5dbf7950d80bb627d8d826fe2303218e5c543879f81aefc1ce75eac313f7f46b701f006a6104aef2
+  metadata.gz: ec260bbe626542c479ab2dc5e3fd78baa44b9cda4b521ae5cfe8f8a07e4c8090c1cae76081cf07e21af13249595e9ebd48dd39ad254a8f94db3ab700e13fb56c
+  data.tar.gz: 23b1e4c40ad25edaa644e920f288a0fb38784a07cf6d327dde7727394f10e6c5fe1eff773c8a89f52d80dfbf47d69cc337f4a9f5bca91dbe20c72463a5bf2139

data/CHANGELOG.md

@@ -0,0 +1,17 @@
+# CHANGELOG
+
+## (develop)
+  - **FIX** Use `:compat` mode when using `Oj` gem to dump/load JSON content.
+  - **FIX** Prevent unnecessary `Fleck::Request` lock for response reception if the response already received.
+  - **NEW** Implemented a timeout functionality for asynchronous request, so that if the request isn't completed within that timeout, it will be canceled and removed from
+            requests list.
+  - **NEW** Set `mandatory: true` when publishing the request to RabbitMQ for both `Fleck::Client` and `Fleck::Consumer`, in order to ensure that requests and responses
+            are enqueued for sure to RabbitMQ.
+  - **NEW** Implemented `#pause` and `#resume` methods for `Fleck::Consumer`, that allows to pause the consumer from consuming messages.
+  - **NEW** `Fleck::Consumer::Response#reject!` support, that allows to reject the processed message. By default `requeue` parameter is set to `false`, so that
+            failed requests aren't requeued. You should call `response.reject(requeue: true)` within the `on_message` method, if you want to requeue the processing
+            message.
+
+## v0.2.0 (18 February 2016)
+  - **NEW** `timeout` (synchronous requests only) and `queue` support for `Fleck::Client#request`
+  - **NEW** Keywords arguments for `Fleck::Client#request` method (ex. `client.request(headers: {h1: v1, ...}, params: {p1: v2, ...}`)

data/example.rb

@@ -15,7 +15,10 @@ end
 connection = Fleck.connection(host: "127.0.0.1", port: 5672, user: user, pass: pass, vhost: "/")
 client = Fleck::Client.new(connection, "example.queue")
 
-count = 0
+count   = 0
+success = 0
+failure = 0
+
 mutex = Mutex.new
 lock  = Mutex.new
 condition = ConditionVariable.new
@@ -24,17 +27,22 @@ class First < Fleck::Consumer
   configure queue: "example.queue", concurrency: CONCURRENCY.to_i
 
   def on_message(request, response)
-    if request.action == "incr"
-      response.body = "#{request.params[:num].to_i + 1}. Hello, World!"
+    if rand > 0.1
+      if request.action == "incr"
+        response.body = "#{request.params[:num].to_i + 1}. Hello, World!"
+      else
+        response.not_found
+      end
     else
-      response.not_found
+      logger.warn "REJECTING REQUEST {headers: #{request.headers}, params: #{request.params}"
+      response.reject!(requeue: true)
     end
   end
 end
 
 Thread.new do
   SAMPLES.times do |i|
-    client.request({action: :incr}, {num: i}, true) do |request, response|
+    client.request(headers: {action: :incr}, params: {num: i}, async: true, timeout: 1) do |request, response|
       if response.status == 200
         request.logger.debug response.body
       else
@@ -42,13 +50,23 @@ Thread.new do
       end
       mutex.synchronize do
         count += 1
+        if response.status == 200
+          success += 1
+        else
+          failure += 1
+        end
+
         lock.synchronize { condition.signal } if count >= SAMPLES
       end
     end
   end
 end
 
+at_exit do
+  puts "Total: #{count}, Success: #{success}, Failure: #{failure}"
+end
+
 lock.synchronize { condition.wait(lock) }
+exit
 
-puts "Total: #{count}"
-First.consumers.map(&:terminate)
+#First.consumers.map(&:terminate)

data/fleck.gemspec

@@ -32,4 +32,5 @@ Gem::Specification.new do |spec|
   spec.add_dependency             "bunny", "~> 2.2"
   spec.add_dependency             "thread_safe", "~> 0.3"
   spec.add_dependency             "oj", "~> 2.14"
+  spec.add_dependency             "ztimer", "~> 0.3"
 end

data/lib/fleck/client/request.rb

@@ -3,25 +3,28 @@ module Fleck
   class Client::Request
     include Fleck::Loggable
 
-    attr_reader :id, :response
+    attr_reader :id, :response, :completed
 
-    def initialize(exchange, routing_key, reply_to, headers = {}, params = {}, &callback)
+    def initialize(client, routing_key, reply_to, headers: {}, params: {}, timeout: nil, &callback)
       @id              = SecureRandom.uuid
       logger.progname += " #{@id}"
 
       logger.debug "Preparing new request"
 
-      @exchange    = exchange
+      @client      = client
       @routing_key = routing_key
       @reply_to    = reply_to
       @params      = params
       @headers     = headers
+      @timeout     = timeout
       @response    = nil
       @lock        = Mutex.new
       @condition   = ConditionVariable.new
       @callback    = callback
       @started_at  = nil
       @ended_at    = nil
+      @completed   = false
+      @async       = false
 
       logger.debug "Request prepared"
     end
@@ -36,20 +39,34 @@ module Fleck
 
     def send!(async = false)
       @started_at = Time.now.to_f
+      @async = async
       data = Oj.dump({
         headers: @headers,
         params:  @params
       }, mode: :compat)
       logger.debug("Sending request with data: #{data}")
 
-      @exchange.publish(data, routing_key: @routing_key, reply_to: @reply_to, correlation_id: @id)
-      @lock.synchronize { @condition.wait(@lock) } unless async
+      options = { routing_key: @routing_key, reply_to: @reply_to, correlation_id: @id, mandatory: true }
+      options[:expiration] = (@timeout * 1000).to_i unless @timeout.nil?
+
+      @client.publish(data, options)
+
+      @lock.synchronize do
+        unless @async || @completed
+          logger.debug("Waiting for response")
+          @condition.wait(@lock)
+          logger.debug("Request terminated.")
+        end
+      end
     end
 
     def complete!
-      @lock.synchronize { @condition.signal }
-      @ended_at = Time.now.to_f
-      logger.debug "Done in #{((@ended_at - @started_at).round(5) * 1000).round(2)} ms"
+      @lock.synchronize do
+        @completed = true
+        @ended_at  = Time.now.to_f
+        logger.debug "Done #{@async ? 'async' : 'synchronized'} in #{((@ended_at - @started_at).round(5) * 1000).round(2)} ms"
+        @condition.signal unless @async
+      end
     end
 
     def cancel!

data/lib/fleck/client/response.rb

@@ -5,7 +5,7 @@ module Fleck
 
     attr_accessor :status, :headers, :body, :errors
     def initialize(payload)
-      @data    = Oj.load(payload).to_hash_with_indifferent_access
+      @data    = Oj.load(payload, mode: :compat).to_hash_with_indifferent_access
       @status  = @data["status"]
       @headers = @data["headers"] || {}
       @body    = @data["body"]

data/lib/fleck/client.rb

@@ -10,15 +10,31 @@ module Fleck
       @exchange    = @channel.default_exchange
       @reply_queue = @channel.queue("", exclusive: true)
       @requests    = ThreadSafe::Hash.new
+      @terminated  = false
+      @mutex       = Mutex.new
+
+      @exchange.on_return do |return_info, metadata, content|
+        begin
+          logger.warn "Request #{metadata[:correlation_id]} returned"
+          request = @requests[metadata[:correlation_id]]
+          if request
+            request.cancel!
+            @requests.delete metadata[:correlation_id]
+          end
+        rescue => e
+          logger.error e.inspect + "\n" + e.backtrace.join("\n")
+        end
+      end
 
       @subscription = @reply_queue.subscribe do |delivery_info, metadata, payload|
         begin
           logger.debug "Response received: #{payload}"
-          request = @requests[metadata[:correlation_id]]
+          request = @requests.delete metadata[:correlation_id]
           if request
             request.response = Fleck::Client::Response.new(payload)
             request.complete!
-            @requests.delete metadata[:correlation_id]
+          else
+            logger.warn "Request #{metadata[:correlation_id]} not found!"
           end
         rescue => e
           logger.error e.inspect + "\n" + e.backtrace.join("\n")
@@ -33,18 +49,31 @@ module Fleck
     end
 
     def request(headers: {}, params: {}, async: false, timeout: nil, queue: @queue_name, &block)
-      request = Fleck::Client::Request.new(@exchange, queue, @reply_queue.name, headers, params, &block)
+      if @terminated
+        return Fleck::Client::Response.new(Oj.dump({status: 503, errors: ['Service Unavailable'], body: nil} , mode: :compat))
+      end
+
+      request = Fleck::Client::Request.new(self, queue, @reply_queue.name, headers: headers, params: params, timeout: timeout, &block)
       @requests[request.id] = request
       if timeout && !async
         begin
           Timeout.timeout(timeout.to_f) do
             request.send!(false)
           end
-        rescue Timeout::Error => e
+        rescue Timeout::Error
           logger.warn "Failed to get any response in #{timeout} seconds for request #{request.id.to_s.color(:red)}! The request will be canceled."
           request.cancel!
           @requests.delete request.id
         end
+      elsif timeout && async
+        request.send!(async)
+        Ztimer.after(timeout * 1000) do |slot|
+          unless request.completed
+            logger.warn "TIMEOUT #{request.id} (#{((slot.executed_at - slot.enqueued_at) / 1000.to_f).round(2)} ms)"
+            request.cancel!
+            @requests.delete request.id
+          end
+        end
       else
         request.send!(async)
       end
@@ -52,17 +81,24 @@ module Fleck
       return request.response
     end
 
+    def publish(data, options)
+      return if @terminated
+      @mutex.synchronize { @exchange.publish(data, options) }
+    end
+
     def terminate
+      @terminated = true
       logger.info "Unsubscribing from #{@reply_queue.name}"
-      @requests.each do |id, request|
+      @subscription.cancel # stop receiving new messages
+      logger.info "Canceling pending requests"
+      # cancel pending requests
+      while item = @requests.shift do
         begin
-          request.cancel!
+          item[1].cancel!
         rescue => e
           logger.error e.inspect + "\n" + e.backtrace.join("\n")
         end
       end
-      @requests.clear
-      @subscription.cancel
     end
   end
 end

data/lib/fleck/consumer/request.rb

@@ -25,7 +25,7 @@ module Fleck
 
     def parse_request!
       logger.debug "Parsing request: #{@payload}"
-      @data    = Oj.load(@payload).to_hash_with_indifferent_access
+      @data    = Oj.load(@payload, mode: :compat).to_hash_with_indifferent_access
       @headers = @data["headers"] || {}
       @action  = @headers["action"]
       @params  = @data["params"] || {}

data/lib/fleck/consumer/response.rb

@@ -9,10 +9,25 @@ module Fleck
       @id = request_id
       logger.progname += " #{@id}"
 
-      @status  = 200
-      @errors  = []
-      @headers = {}
-      @body    = nil
+      @status   = 200
+      @errors   = []
+      @headers  = {}
+      @body     = nil
+      @rejected = false
+      @requeue  = false
+    end
+
+    def reject!(requeue: false)
+      @rejected = true
+      @requeue  = requeue
+    end
+
+    def rejected?
+      return @rejected
+    end
+
+    def requeue?
+      return @requeue
     end
 
     def not_found(msg = nil)

data/lib/fleck/consumer.rb

@@ -45,6 +45,7 @@ module Fleck
     def initialize(thread_id = nil)
       @__thread_id    = thread_id
       @__connection   = nil
+      @__consumer_tag = nil
 
       @__host       = configs[:host]
       @__port       = configs[:port]
@@ -69,6 +70,7 @@ module Fleck
     end
 
     def terminate
+      pause
       unless @__channel.closed?
         @__channel.close
         logger.info "Consumer successfully terminated."
@@ -87,6 +89,35 @@ module Fleck
       @configs ||= self.class.configs
     end
 
+    def connection
+      return @__connection
+    end
+
+    def channel
+      return @__channel
+    end
+
+    def queue
+      return @__queue
+    end
+
+    def exchange
+      return @__exchange
+    end
+
+    def subscription
+      return @__subscription
+    end
+
+    def pause
+      cancel_ok = @__subscription.cancel
+      @__consumer_tag = cancel_ok.consumer_tag
+    end
+
+    def resume
+      subscribe!
+    end
+
     protected
 
     def connect!
@@ -108,7 +139,11 @@ module Fleck
 
     def subscribe!
       logger.debug "Consuming from queue: #{@__queue_name.color(:green)}"
-      @__subscription = @__queue.subscribe do |delivery_info, metadata, payload|
+
+      options = { manual_ack: true }
+      options[:consumer_tag] = @__consumer_tag if @__consumer_tag
+
+      @__subscription = @__queue.subscribe(options) do |delivery_info, metadata, payload|
         response = Fleck::Consumer::Response.new(metadata.correlation_id)
         begin
           request  = Fleck::Consumer::Request.new(metadata, payload)
@@ -124,8 +159,19 @@ module Fleck
           response.errors << 'Internal Server Error'
         end
 
-        logger.debug "Sending response: #{response}"
-        @__exchange.publish(response.to_json, routing_key: metadata.reply_to, correlation_id: metadata.correlation_id)
+        if response.rejected?
+          # the request was rejected, so we have to notify the reject
+          logger.warn "Request #{response.id} was rejected!"
+          @__channel.reject(delivery_info.delivery_tag, response.requeue?)
+        else
+          logger.debug "Sending response: #{response}"
+          if @__channel.closed?
+            logger.warn "Channel already closed! The response #{metadata.correlation_id} is going to be dropped."
+          else
+            @__exchange.publish(response.to_json, routing_key: metadata.reply_to, correlation_id: metadata.correlation_id, mandatory: true)
+            @__channel.ack(delivery_info.delivery_tag)
+          end
+        end
       end
     end
 

data/lib/fleck/version.rb

@@ -1,3 +1,3 @@
 module Fleck
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

data/lib/fleck.rb

@@ -5,6 +5,7 @@ require "bunny"
 require "thread_safe"
 require "securerandom"
 require "oj"
+require "ztimer"
 require "fleck/version"
 require "fleck/hash_with_indifferent_access"
 require "fleck/configuration"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fleck
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Groza Sergiu
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-02-18 00:00:00.000000000 Z
+date: 2016-04-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -108,6 +108,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.14'
+- !ruby/object:Gem::Dependency
+  name: ztimer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
 description: "\n    Fleck is a library for syncronous and asyncronous communication
   over Message Queues services. Unlike a common\n    HTTP communication, Fleck requests
   and responses are pure JSON messages.\n  "
@@ -120,6 +134,7 @@ files:
 - ".gitignore"
 - ".rspec"
 - ".travis.yml"
+- CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
 - LICENSE.txt