gorg_service 0.1.2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3ef9f94b4bf945e579b3cd7e41bee32b4daa667a
-  data.tar.gz: 0526af44cef52f9a89727b929ed5df8f0400711e
+  metadata.gz: c367d16bc5790468c190d509d84e89effef69fbb
+  data.tar.gz: 1951f35a70ffacddb8f1f141df9febeadc890420
 SHA512:
-  metadata.gz: a8983e511b243a7f371646633c3fb3b19fdc96dbfcf2278296155f915bfd5c8dda3d13aa0cd30863692390532c012097ba11a447f1cd97f8ba1eb34b95e33076
-  data.tar.gz: 5ed51cc437c9a347c7b9958a791a55f4605b52234c816f28bfda5709e379de91eb707a68a77245b91afcba70e5e0baf9a8f05a3d9ab292139be366cc200798ef
+  metadata.gz: 912bc14211df2c260d360023f3e6ee378bb228db615408d46615197bb0db4f0acaa4957aeb10bb1d3ba1c1a2b6e7593fbbb12b99a50467583fce7326b22028b4
+  data.tar.gz: 7c1bc8ac7f5f3268d35b5d481efe7fb2821b5d2abb6fdbfb3389a47c2de453f1b6660bfdd1e267ba4cbbba90318953b4c2e9bf313e8b3a0ef51c8dc430680d2e

data/.gitignore

@@ -9,4 +9,5 @@
 /tmp/
 /build/
 *.gem
-/bin/worker
+/bin/worker
+/spec/gorg_service/support/conf/rabbit_mq.yml

data/.todo.reek

@@ -0,0 +1,26 @@
+---
+Attribute:
+  exclude:
+  - GorgService#configuration
+  - GorgService::Configuration#application_id
+  - GorgService::Configuration#application_name
+  - GorgService::Configuration#message_handler_map
+  - GorgService::Configuration#rabbitmq_deferred_time
+  - GorgService::Configuration#rabbitmq_exchange_name
+  - GorgService::Configuration#rabbitmq_host
+  - GorgService::Configuration#rabbitmq_max_attempts
+  - GorgService::Configuration#rabbitmq_password
+  - GorgService::Configuration#rabbitmq_port
+  - GorgService::Configuration#rabbitmq_queue_name
+  - GorgService::Configuration#rabbitmq_user
+  - GorgService::Configuration#rabbitmq_vhost
+  - GorgService::Message#data
+  - GorgService::Message#errors
+  - GorgService::Message#event
+  - GorgService::Message#id
+TooManyInstanceVariables:
+  exclude:
+  - GorgService::Configuration
+ControlParameter:
+  exclude:
+  - GorgService::Message#initialize

data/.travis.yml

@@ -4,4 +4,9 @@ rvm:
 before_install: gem install bundler -v 1.11.2
 addons:
     code_climate:
-        repo_token: 8a2a969cd1d481fbc952943cc27b54584b2f27217ea2e2905e77652d44c502be
+        repo_token: 8a2a969cd1d481fbc952943cc27b54584b2f27217ea2e2905e77652d44c502be
+before_script:
+  - cp spec/gorg_service/support/conf/rabbit_mq.yml.travis spec/gorg_service/support/conf/rabbit_mq.yml
+sudo: required
+services:
+  - rabbitmq

data/Gemfile

@@ -2,3 +2,4 @@ source 'https://rubygems.org'
 
 gemspec
 gem "codeclimate-test-reporter", group: :test, require: nil
+gem 'reek', '~> 4.0'

data/README.md

@@ -1,5 +1,6 @@
 # GorgService
-[![Code Climate](https://codeclimate.com/github/Zooip/gorg_service/badges/gpa.svg)](https://codeclimate.com/github/Zooip/gorg_service) [![Test Coverage](https://codeclimate.com/github/Zooip/gorg_service/badges/coverage.svg)](https://codeclimate.com/github/Zooip/gorg_service/coverage) [![Build Status](https://travis-ci.org/Zooip/gorg_service.svg?branch=master)](https://travis-ci.org/Zooip/gorg_service)
+[![Code Climate](https://codeclimate.com/github/Zooip/gorg_service/badges/gpa.svg)](https://codeclimate.com/github/Zooip/gorg_service) [![Test Coverage](https://codeclimate.com/github/Zooip/gorg_service/badges/coverage.svg)](https://codeclimate.com/github/Zooip/gorg_service/coverage) [![Build Status](https://travis-ci.org/Zooip/gorg_service.svg?branch=master)](https://travis-ci.org/Zooip/gorg_service) [![Gem Version](https://badge.fury.io/rb/gorg_service.svg)](https://badge.fury.io/rb/gorg_service) [![Dependency Status](https://gemnasium.com/badges/github.com/Zooip/gorg_service.svg)](https://gemnasium.com/github.com/Zooip/gorg_service)
+Standard RabbitMQ bot used in Gadz.org SOA
 
 ## Installation
 
@@ -19,7 +20,7 @@ Or install it yourself as:
 
 ## Setup
   
-  Before being used GramV1Client must be configured. In Rails app, put it in an Initializer.
+  Before being used, GorgService must be configured. In Rails app, put it in an Initializer.
 
 ```ruby
 GorgService.configure do |c|
@@ -45,18 +46,97 @@ GorgService.configure do |c|
   # c.rabbitmq_queue_name = c.application_name
   # c.rabbitmq_exchange_name = "exchange"
   #
-  # time before trying again on softfail (temporary error)
+  # time before trying again on softfail in milliseconds (temporary error)
   # c.rabbitmq_deferred_time = 1800000 # 30min
   # 
   # maximum number of try before discard a message
   # c.rabbitmq_max_attempts = 48 # 24h with default deferring delay
+  #
+  # Routing hash
+  #  map routing_key of received message with MessageHandler 
+  #  exemple:
+  # c.message_handler_map={
+  #   "some.routing.key" => MyMessageHandler,
+  #   "Another.routing.key" => OtherMessageHandler,
+  #   "third.routing.key" => MyMessageHandler,
+  # }
+  c.message_handler_map= {} #TODO : Set my routing hash
 
  end
 ```
 
 ## Usage
 
-TODO: Write usage instructions here
+To start the RabbitMQ consummer use :
+```ruby
+my_service = GorgService.new
+my_service.run
+```
+### Routing and MessageHandler
+When running, GorgService act as a consumer on Gadz.org RabbitMQ network.
+It bind its queue on the main exchange and subscribes to routing keys defines in `message_handler_map`
+
+Each received message will be routed to the corresponding `MessageHandler`
+> **Warning** : RabbitMQ wildcards characters `#` and `*`are NOT supported for now
+
+A `MessageHandler` is a kind of controller. This is where you put the message is processed.
+A `MessageHandler` expect a `GorgService::Message` as param of its `initializer`method.
+
+Here is an exemple `MessageHandler` :
+```ruby
+require 'json'
+require 'json-schema' #Checkout https://github.com/ruby-json-schema/json-schema
+
+class ExampleMessageHandler < GorgService::MessageHandler
+
+  EXPECTED_SCHEMA = {
+    "type" => "object",
+    "required" => ["user_id"],
+    "properties" => {
+      "user_id" => {"type" => "integer"}
+    }
+  }
+  
+  def initialize(msg)
+    data=msg.data
+    begin
+      JSON::Validator.validate!(EXPECTED_SCHEMA, data)
+    rescue JSON::Schema::ValidationError => e
+      #This message can't be processed, it will be discarded
+      raise_hardfail("Invalid message",e)
+    end
+
+    begin
+      MyAPI.send(msg.data)
+    rescue MyAPI::UnavailableConnection => e
+      # This message can be processed but external resources
+      # are not available at this moment, retry later
+      raise_softfail("Can't connect to MyAPI",e)
+    end
+  end
+end
+```
+
+As showed in this example,`GorgService::MessageHandler` provides 2 helpers :
+
+ - `raise_hardfail`: This will raise a `HardfailError`exception. The message can't be processed and will never be. It is logged and send back to main exchange for audit purpose.
+
+ - `raise_softfail`: This will raise a `SoftfailError`exception. The message can't be processed at this moment but may be processed in future : connection problems, rate limiting API, etc. It is sent to a deferred queue where it will be delayed for `rabbitmq_deferred_time`millisecconds before being sent back in the main exchange for re-queuing.
+
+Each one of this helpers expect two params :
+
+ - `message` : The information to be displayed in message's error log
+ - `exception` (optional) : The runtime esception causing the hardfail, for debug purpose
+
+### Message structure
+`GorgService::Message` is defined [here](https://github.com/Zooip/gorg_service/blob/master/lib/gorg_service/message.rb)
+
+It provides 4 attributes :
+
+ - `event` : this is the same as the routing key
+ - `id`: message UUID
+ - `errors`: `Hash` containing the message error log with previous errors
+ - `data`: `Hash` containing the data to be processed
 
 ## Development
 

data/gorg_service.gemspec

@@ -31,4 +31,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 1.11"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "codeclimate-test-reporter", "~> 3.0"
+  spec.add_development_dependency 'bogus', '~> 0.1.6'
+  spec.add_development_dependency 'bunny-mock', '~> 1.4'
+  spec.add_development_dependency 'byebug', '~> 9.0'
 end

data/lib/gorg_service/configuration.rb

@@ -1,3 +1,4 @@
+# Add configuration features to GorgService
 class GorgService
   class << self
     attr_writer :configuration
@@ -8,10 +9,12 @@ class GorgService
 
 
     def configure
+      @configuration = Configuration.new
       yield(configuration)
     end
   end
 
+  # Hold configuration of GorgService in instance variables
   class Configuration
     attr_accessor :application_name,
                   :application_id,
@@ -23,11 +26,11 @@ class GorgService
                   :rabbitmq_max_attempts,
                   :rabbitmq_user,
                   :rabbitmq_password,
-                  :message_handler_map,
+                  :rabbitmq_vhost,
+                  :message_handler_map
 
 
     def initialize
-
       @application_name        = "GorgService"
       @application_id          = "gs" 
       @message_handler_map     = {}
@@ -38,6 +41,7 @@ class GorgService
       @rabbitmq_exchange_name  = "exchange"
       @rabbitmq_user           = nil
       @rabbitmq_password       = nil
+      @rabbitmq_vhost          = "/"
       @rabbitmq_max_attempts   = 48         #24h with default timeout
     end
   end

data/lib/gorg_service/errors.rb

@@ -2,17 +2,36 @@
 # encoding: utf-8
 
 class GorgService
+
+  #Common behavior of failling errors
   class FailError < StandardError
     attr_reader :error_raised
 
-    def initialize(error_raised)
+    def initialize(message = nil, error_raised = nil)
+      @message = message
       @error_raised = error_raised
     end
+
+    def message
+      @message
+    end
+
+    def type
+      ""
+    end
   end
 
+  #Softfail error : This message should be processed again later
   class SoftfailError < FailError
+    def type
+      "softfail"
+    end
   end
 
+  #Hardfail error : This message is not processable and will never be
   class HardfailError < FailError
+    def type
+      "hardfail"
+    end
   end
 end

data/lib/gorg_service/listener.rb

@@ -6,16 +6,13 @@ require "bunny"
 class GorgService
   class Listener
 
-    def initialize(host: "localhost", port: 5672, queue_name: "gapps", rabbitmq_user: nil, rabbitmq_password: nil, exchange_name: nil, message_handler_map: {default: DefaultMessageHandler}, deferred_time: 1800000)
-      @host=host
-      @port=port
+    def initialize(bunny_session: nil,queue_name: "gapps", exchange_name: nil, message_handler_map: {default: DefaultMessageHandler}, deferred_time: 1800000, max_attempts: 48)
       @queue_name=queue_name
       @exchange_name=exchange_name
-      @rabbitmq_user=rabbitmq_user
-      @rabbitmq_password=rabbitmq_password
       @message_handler_map=message_handler_map
       @deferred_time=deferred_time
-
+      @max_attempts=max_attempts
+      @rmq_connection=bunny_session
     end
 
     def listen
@@ -31,52 +28,55 @@ class GorgService
 
       q.bind(@exchange_name, :routing_key => '#')
 
-      puts " [*] Waiting for messages in #{q.name}. To exit press CTRL+C"
       ch.prefetch(1)
 
-      begin
-        q.subscribe(:manual_ack => true, :block => true) do |delivery_info, properties, body|
-          routing_key=delivery_info[:routing_key]
-          puts " [#] Received message with routing key #{routing_key} containing : #{body}"
-          message_handler=message_handler_for routing_key
-          message=Message.parse_body(body)
-          begin
-            if message_handler
-              message_handler.new(message)
-            else
-              raise HardfailError.new(), "Unknown routing_key"
-            end
-          rescue SoftfailError => e
-            message.log_error(e)
-            puts " [*] SOFTFAIL ERROR : #{e.message}"
-            if message.errors.count > 5
-              puts " [*] DISCARD MESSAGE : #{message.errors.count} errors in message log"
-            else
-
-              send_to_deferred_queue(message)
-            end
-          rescue HardfailError => e
-             puts " [*] SOFTFAIL ERROR : #{e.message}"
-             puts " [*] DISCARD MESSAGE"          
-          end
-
-          ch.ack(delivery_info.delivery_tag)
-        end
-      rescue Interrupt => _
-        conn.close
+      q.subscribe(:manual_ack => true) do |delivery_info, properties, body|
+        routing_key=delivery_info[:routing_key]
+        puts " [#] Received message with routing key #{routing_key} containing : #{body}"
+        message_handler=message_handler_for routing_key
+        message=Message.parse_body(body)
+
+        call_message_handler(message_handler, message)
+
+        ch.ack(delivery_info.delivery_tag)
       end
+
     end
 
     def rmq_connection
-        if @rmq_connection
-          @rmq_connection
+      @rmq_connection.start unless @rmq_connection.connected?
+      @rmq_connection
+    end
+
+    def call_message_handler(message_handler, message)
+      begin
+        raise HardfailError.new(), "Routing error" unless message_handler
+          message_handler.new(message)
+
+      rescue SoftfailError => e
+        message.log_error(e)
+        process_softfail(e,message)
+
+      rescue HardfailError => e
+        message.log_error(e)
+        process_hardfail(e)      
+      end
+    end
+
+    def process_softfail(e,message)
+        puts " [*] SOFTFAIL ERROR : #{e.message}"
+        if message.errors.count >= @max_attempts
+          puts " [*] DISCARD MESSAGE : #{message.errors.count} errors in message log"
         else
-          @rmq_connection=Bunny.new(:hostname => @host, :user => @rabbitmq_user, :pass => @rabbitmq_password)
-          @rmq_connection.start
-          @rmq_connection
+          send_to_deferred_queue(message)
         end
     end
 
+    def process_hardfail(e)
+      puts " [*] SOFTFAIL ERROR : #{e.message}"
+      puts " [*] DISCARD MESSAGE"    
+    end
+
     def send_to_deferred_queue(msg)
       conn=rmq_connection
       @delayed_chan||=conn.create_channel
@@ -89,7 +89,7 @@ class GorgService
           }
         )
       puts " [*] DEFER MESSAGE : message sent to #{@queue_name}_deferred qith routing key #{msg.event}"
-      q.publish(msg.to_str, :routing_key => msg.event)
+      q.publish(msg.to_json, :routing_key => msg.event)
     end
 
     def message_handler_for routing_key

data/lib/gorg_service/message.rb

@@ -23,16 +23,8 @@ class GorgService
       @errors=errors
     end
 
-    # Generate new id
-    #
-    # TODO
-    #  Avoid using AppConfig
-    def generate_id 
-      "#{GorgService.configuration.application_id}_#{Time.now.to_i}"
-    end
-
     # Generate RabbitMQ message body
-    def to_str
+    def to_json
       body={
         id: @id,
         event: @event,
@@ -48,7 +40,7 @@ class GorgService
     # Log FailError in message body
     def log_error error
       errors<<{
-            type: error.class.to_s.downcase,
+            type: error.type.downcase,
             message: error.message,
             timestamp: Time.now.utc.iso8601,
             extra: error.error_raised.inspect,
@@ -66,15 +58,19 @@ class GorgService
       begin
         json_body=JSON.parse(body)
 
-        self.new(
+        msg=self.new(
             id: json_body["id"],
             event: json_body["event"],
             data: convert_keys_to_sym(json_body["data"]),
-            errors: convert_keys_to_sym(json_body["errors"]),
+            errors: json_body["errors"]&&json_body["errors"].map{|e| convert_keys_to_sym(e)},
           )
 
+        msg.errors=msg.errors.each do |e|
+         e[:timestamp]=(e[:timestamp] ? DateTime.parse(e[:timestamp]) : nil)
+        end
+        msg
       rescue JSON::ParserError => e
-        raise HardfailError(e), "Unprocessable message : Unable to parse JSON message body"
+        raise GorgService::HardfailError.new(e), "Unprocessable message : Unable to parse JSON message body"
       end
     end
 
@@ -91,6 +87,14 @@ class GorgService
       end
       s2s[input_hash]
     end
+
+    private
+
+    # Generate new id
+    def generate_id 
+      "#{GorgService.configuration.application_id}_#{Time.now.to_i}"
+    end
+
    
 
   end

data/lib/gorg_service/version.rb

@@ -2,5 +2,5 @@
 # encoding: utf-8
 
 class GorgService
-  VERSION = "0.1.2"
+  VERSION = "0.2.0"
 end

data/lib/gorg_service.rb

@@ -6,20 +6,44 @@ require "gorg_service/message"
 require "gorg_service/message_handler"
 
 class GorgService
-  def initialize()
-    @listener=Listener.new(
+  def initialize(listener: nil, bunny_session: nil)
+    
+    @bunny_session= bunny_session || Bunny.new(
+      :hostname => GorgService.configuration.rabbitmq_host,
+      :port => GorgService.configuration.rabbitmq_port,
+      :user => GorgService.configuration.rabbitmq_user,
+      :pass => GorgService.configuration.rabbitmq_password,
+      :vhost => GorgService.configuration.rabbitmq_vhost
+      )
+
+    @listener= listener || Listener.new(
+      bunny_session: @bunny_session,
       message_handler_map:GorgService.configuration.message_handler_map,
-      host: GorgService.configuration.rabbitmq_host,
-      port: GorgService.configuration.rabbitmq_port,
       queue_name: GorgService.configuration.rabbitmq_queue_name,
       exchange_name: GorgService.configuration.rabbitmq_exchange_name,
-      rabbitmq_user: GorgService.configuration.rabbitmq_user,
-      rabbitmq_password: GorgService.configuration.rabbitmq_password,
       deferred_time: GorgService.configuration.rabbitmq_deferred_time,
+      max_attempts: GorgService.configuration.rabbitmq_max_attempts,
       )
   end
 
   def run
+    begin
+      self.start
+      puts " [*] Waiting for messages. To exit press CTRL+C"
+      loop do
+        sleep(1)
+      end
+    rescue SystemExit, Interrupt => _
+      self.stop
+    end
+  end
+
+  def start
+    @bunny_session.start
     @listener.listen
   end
+
+  def stop
+    @bunny_session.close
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gorg_service
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
 platform: ruby
 authors:
 - Alexandre Narbonne
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-05-27 00:00:00.000000000 Z
+date: 2016-05-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bunny
@@ -72,6 +72,62 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: codeclimate-test-reporter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: bogus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.6
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.6
+- !ruby/object:Gem::Dependency
+  name: bunny-mock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '9.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '9.0'
 description: 
 email:
 - alexandre.narbonne@gadz.org
@@ -81,6 +137,7 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".rspec"
+- ".todo.reek"
 - ".travis.yml"
 - Gemfile
 - LICENSE.txt