eventhub-processor2 1.1.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: '037081a3ac65841f8c3efd865fc17e26eaafbe44'
-  data.tar.gz: c38fc7169a5368e381c89595292916d0f9368a1b
+  metadata.gz: 560975d638bb453888cd1c153ef65b8fd186b216
+  data.tar.gz: 3e72d08346c2b3668c17ad4218c0a0e05dc76580
 SHA512:
-  metadata.gz: '081046afc2e55110f336eff3982cfbd1893849d125ff41bb3a75bfe8ad1027342964d05ffee90c25549a293d9a1a1ba6b90d76d8beea1596d4fc8b14cfc7da03'
-  data.tar.gz: f40bb241e4f25f6d12fa2715a2ab99830f56a5ee0afcc468ab8eba8de1ae8f71bfe20de0ac8918ccefa8c93a0feff454752703d6d9c7ac75f72573697425fa1b
+  metadata.gz: 3f5b88b6a0143ee9d89830a8b02f91d5d76b3c95355ab1f6b8fef4cb1f3ae73bab64f199922208ec24868a4e1d40552f0dd67508e4d75287a9cf993b04e949dc
+  data.tar.gz: 393727e798721fea7a9a8c6fd3fb605f98efd64abdb956de2578558597f68f557ff6a4bd6b085277eb1373ab5b071c547860ce86889ffc30b4cfa462f52cdcb0

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    eventhub-processor2 (1.1.0)
+    eventhub-processor2 (1.2.0)
       bunny (~> 2.9)
       celluloid (~> 0.17)
       eventhub-components (~> 0.2)
@@ -73,4 +73,4 @@ DEPENDENCIES
   simplecov (~> 0.15)
 
 BUNDLED WITH
-   1.16.0
+   1.16.1

data/README.md

@@ -5,7 +5,13 @@
 
 # EventHub::Processor2
 
-Next generation gem to build ruby based eventhub processor.
+Next generation gem to build ruby based eventhub processors. Implementation is based on Celluloid, an Actor-based concurrent object framework for Ruby https://celluloid.io. The main idea is to have sub-components in your application and have them supervised and automatically re-booted when they crash.
+
+Processor2 has currently the following sub-components implemented
+* Heartbeater - send hearbeats to EventHub dispatcher every x minutes
+* Publisher - responsible for message publishing
+* Watchdog - Checks regularly broker connection and defined listener queue(s)
+* Listener - Listens to defined queues, parses recevied message into a EventHub::Message instance and calls handle_message method as defined in derived class.
 
 ## Installation
 
@@ -26,6 +32,125 @@ Or install it yourself as:
 
 ## Usage
 
+Create example.rb
+
+```ruby
+module EventHub
+  class Example < Processor2
+
+    def version
+      '1.0.0' # define your version
+    end
+
+    def handle_message(message, args = {})
+      # deal with your parsed EventHub message
+      # message.class => EventHub::Message
+      puts message.process_name # or whatever you need to do
+
+      # args is a hash with currently following keys
+      # => :queue_name (used when listening to multiple queues)
+      # => :content_type
+      # => :priority
+      # => :delivery_tag
+
+      # if an exception is raised in your code
+      # it will be automatically catched by
+      # the processor2 gem and returned
+      # to the event_hub.inbound queue
+
+      # at the end return one of
+      message # return message if sucessfull processing
+
+      # or if you have multiple messages to return to event_hub.inbound queue
+      [ message, new_message1, new_message2]
+
+      # or if there is no message to return to event_hub.inbound queue
+      nil # [] works as well
+    end
+  end
+end
+
+# start your processor instance
+EventHub::Example.new.start
+```
+
+Start your processor and pass optional arguments
+```bash
+bundle exec ruby example.rb --help
+Usage: example [options]
+    -e, --environment ENVIRONMENT    Define environment (default development)
+    -d, --detached                   Run processor detached as a daemon
+    -c, --config CONFIG              Define configuration file
+
+
+bundle exec ruby example.rb
+I, [2018-02-09T15:22:35.649646 #37966]  INFO -- : example (1.1.0): has been started
+I, [2018-02-09T15:22:35.652592 #37966]  INFO -- : Heartbeat is starting...
+I, [2018-02-09T15:22:35.657200 #37966]  INFO -- : Publisher is starting...
+I, [2018-02-09T15:22:35.657903 #37966]  INFO -- : Watchdog is starting...
+I, [2018-02-09T15:22:35.658336 #37966]  INFO -- : Running watchdog...
+I, [2018-02-09T15:22:35.658522 #37966]  INFO -- : Listener is starting...
+I, [2018-02-09T15:22:35.699161 #37966]  INFO -- : Listening to queue [example]
+```
+
+## Configuration
+
+If --config option is not provided processor tries to load config/{class_name}.json. If file does not exist it loads default values as specified below.
+
+```json
+{
+  "development": {
+    "server": {
+      "user": "guest",
+      "password": "guest",
+      "host": "localhost",
+      "vhost": "event_hub",
+      "port": 5672,
+      "tls": false,
+      "tls_cert": null,
+      "tls_key": null,
+      "tls_ca_certificates": [],
+      "verify_peer": false,
+      "show_bunny_logs": false
+    },
+    "processor": {
+      "listener_queues": [
+        "{class_name}"
+      ],
+      "heartbeat_cycle_in_s": 300,
+      "watchdog_cycle_in_s": 15,
+      "restart_in_s": 15
+    }
+  }
+}
+```
+
+Feel free to define additional hash key/values (outside of server and processor key) as required by your application.
+
+```json
+{
+  "development": {
+    "server": {
+    },
+    "processor": {
+    },
+    "database": {
+      "user": "guest",
+      "password": "secret",
+      "name": {
+        "subname": "value"
+      }
+    }
+  }
+}
+```
+
+```ruby
+  # access configuration values in your application as follows
+  Configuration.processor.database[:user]           # => "guest"
+  Configuration.processor.database[:password]       # => "secret"
+  Configuration.processor.database[:name][:subname] # => "value"
+```
 
 ## Development
 

data/example/README.md

@@ -2,9 +2,9 @@
 
 ### Description
 
-Example application is a suite of applicaitons in order to test reliabiliy and performance of processor2 gem.
+Example folder contains a series of applications in order to test reliability and performance of processor2 gem.
 
-How does it work?
+### How does it work?
 
 A message is passed throuhg the following components.
 publisher.rb => [example.outbound] => router.rb => [example.inbound] => receiver.rb
@@ -15,16 +15,22 @@ publisher.rb => [example.outbound] => router.rb => [example.inbound] => receiver
 
 3. receiver.rb gets the message and deletes the file with the given ID
 
-Goal: What ever happens to these components (restarted, killed and restarted, stopped and started, message broker restarted) if you do a graceful shutdown at the end there should be no message in the /data folder.
+### Goal
+What ever happens to these components (restarted, killed and restarted, stopped and started, message broker killed, stopped and started) if you do a graceful shutdown at the end there should be no message in the /data folder (except store.json).
 
-Graceful shutdown: Stop producer.rb. Leave the other components running until all messages in example.* queues are gone. Stop remaining components.
+Graceful shutdown with CTRL-C or TERM signal to pdi
+* Stop producer.rb. Leave the other components running until all messages in example.* queues are gone.
+* Stop remaining components
+* Check ./example/data folder
 
 
-### How to use
-* Make sure docker container (process-rabbitmq) is running
-* Start one or more router.rb
-* Start one or more receier.rb
-* Start one or more publisher.rb
-* Start crasher.rb if you like (or do it manually)
+### How to use?
+* Make sure docker container (process-rabbitmq) is running (see [readme](../docker/README.md))
+* Start one or more router with: bundle exec ruby router.rb
+* Start one or more receiver with: bundle exec ruby receier.rb
+* Start one publisher with: bundle exec ruby publisher.rb
+* Start one crasher with: bundle exec ruby crasher.rb (or do this manually)
 
 ### Note
+* Publisher has a simple transaction store implemented to deal with issues between file creation and file publishing. At the end of the publisher process in the cleanup method pending transaction get processed and coresponding files get deleted.
+* Watch for huge log files!

data/example/publisher.rb

@@ -5,8 +5,17 @@ require 'securerandom'
 require 'eventhub/components'
 require_relative '../lib/eventhub/sleeper'
 
-# Example module
-module Example
+SIGNALS_FOR_TERMINATION = [:INT, :TERM, :QUIT]
+SIGNALS_FOR_RELOAD_CONFIG = [:HUP]
+ALL_SIGNALS = SIGNALS_FOR_TERMINATION + SIGNALS_FOR_RELOAD_CONFIG
+PAUSE_BETWEEN_WORK = 0.05 # default is 0.05
+
+Celluloid.logger = nil
+Celluloid.exception_handler { |ex| Publisher.logger.error "Exception occured: #{ex}}" }
+
+# Publisher module
+module Publisher
+
   def self.logger
     unless @logger
       @logger = ::EventHub::Components::MultiLogger.new
@@ -17,178 +26,185 @@ module Example
     end
     @logger
   end
-end
 
-SIGNALS_FOR_TERMINATION = [:INT, :TERM, :QUIT]
-SIGNALS_FOR_RELOAD_CONFIG = [:HUP]
-ALL_SIGNALS = SIGNALS_FOR_TERMINATION + SIGNALS_FOR_RELOAD_CONFIG
+  # Store to track pending files (files not yet confirmed to be sent)
+  class TransactionStore
+    include Celluloid
+    finalizer :cleanup
 
-Celluloid.logger = nil
-Celluloid.exception_handler { |ex| Example.logger.error "Exception occured: #{ex}}" }
-
-# Store to track pending files (files not yet confirmed to be sent)
-class TransactionStore
-  include Celluloid
-  finalizer :cleanup
-
-  def initialize
-    @filename = 'data/store.json'
-    if File.exist?(@filename)
-      cleanup
-    else
-      File.write(@filename, '{}')
-    end
-  end
+    def initialize
+      @start = Time.now
+      @files_sent = 0
 
-  def start(name)
-    store = read_store
-    store[name] = Time.now.strftime('%Y-%m-%d %H:%M:%S.%L')
-    write_store(store)
-  end
+      @filename = 'data/store.json'
+      if File.exist?(@filename)
+        cleanup
+      else
+        File.write(@filename, '{}')
+      end
 
-  def stop(name)
-    store = read_store
-    store.delete(name)
-    write_store(store)
-  end
+      every(30) { write_statistics }
+    end
 
-  def cleanup
-    # cleanup pending entries
-    Example.logger.info("Cleaning pending transactions...")
-    store = read_store
-    store.keys.each do |name|
-      name = "data/#{name}.json"
-      if File.exist?(name)
-        File.delete(name)
-        Example.logger.info("Deleted: #{name}")
-      end
+    def start(name)
+      store = read_store
+      store[name] = Time.now.strftime('%Y-%m-%d %H:%M:%S.%L')
+      write_store(store)
     end
-    write_store({})
-  end
 
-  private
-    def read_store
-      JSON.parse(File.read(@filename))
+    def stop(name)
+      store = read_store
+      store.delete(name)
+      write_store(store)
+      @files_sent += 1
     end
 
-    def write_store(store)
-      File.write(@filename, store.to_json)
+    def cleanup
+      # cleanup pending entries
+      Publisher.logger.info("Cleaning pending transactions...")
+      store = read_store
+      store.keys.each do |name|
+        name = "data/#{name}.json"
+        if File.exist?(name)
+          File.delete(name)
+          Publisher.logger.info("Deleted: #{name}")
+        end
+      end
+      write_store({})
+      write_statistics
     end
-end
 
-# Publisher
-class Publisher
-  include Celluloid
+    def write_statistics
+      now = Time.now
+      rate = @files_sent / (now-@start)
+      time_spent = (now-@start)/60
+      Publisher.logger.info("Started @ #{@start.strftime('%Y-%m-%d %H:%M:%S.%L')}: Files sent within #{'%0.1f' % time_spent} minutes: #{@files_sent}, #{ '%0.1f' % rate} files/second")
+    end
 
-  def initialize
-    async.start
+    private
+      def read_store
+        JSON.parse(File.read(@filename))
+      end
+
+      def write_store(store)
+        File.write(@filename, store.to_json)
+      end
   end
 
-  def start
-    connect
-    loop do
-      do_the_work
-      sleep 0.050
+  # Worker
+  class Worker
+    include Celluloid
+
+    def initialize
+      async.start
     end
-  ensure
-    @connection.close if @connection
-  end
 
-  private
+    def start
+      connect
+      loop do
+        do_the_work
+        sleep PAUSE_BETWEEN_WORK
+      end
+    ensure
+      @connection.close if @connection
+    end
 
-  def connect
-    @connection = Bunny.new(vhost: 'event_hub',
-                            automatic_recovery: false,
-                            logger: Logger.new('/dev/null'))
-    @connection.start
-    @channel = @connection.create_channel
-    @channel.confirm_select
-    @exchange = @channel.direct('example.outbound', durable: true)
-  end
+    private
 
-  def do_the_work
-    #prepare id and content
-    id = SecureRandom.uuid
-    file_name = "data/#{id}.json"
-    data = { body: { id: id } }.to_json
-
-    # start transaction...
-    Celluloid::Actor[:transaction_store].start(id)
-    File.write(file_name, data)
-    Example.logger.info("[#{id}] - Message/File created")
-
-    @exchange.publish(data, persistent: true)
-    success = @channel.wait_for_confirms
-    if success
-      Celluloid::Actor[:transaction_store].stop(id) if Celluloid::Actor[:transaction_store]
-      Example.logger.info("[#{id}] - Message sent")
-    else
-      Example.logger.error("[#{id}] -  Published message not confirmed")
+    def connect
+      @connection = Bunny.new(vhost: 'event_hub',
+                              automatic_recovery: false,
+                              logger: Logger.new('/dev/null'))
+      @connection.start
+      @channel = @connection.create_channel
+      @channel.confirm_select
+      @exchange = @channel.direct('example.outbound', durable: true)
     end
-  end
-end
 
-# Application
-class Application
-  def initialize
-    @sleeper = EventHub::Sleeper.new
-    @command_queue = []
+    def do_the_work
+      #prepare id and content
+      id = SecureRandom.uuid
+      file_name = "data/#{id}.json"
+      data = { body: { id: id } }.to_json
+
+      # start transaction...
+      Celluloid::Actor[:transaction_store].start(id)
+      File.write(file_name, data)
+      Publisher.logger.info("[#{id}] - Message/File created")
+
+      @exchange.publish(data, persistent: true)
+      success = @channel.wait_for_confirms
+      if success
+        Celluloid::Actor[:transaction_store].stop(id) if Celluloid::Actor[:transaction_store]
+        Publisher.logger.info("[#{id}] - Message sent")
+      else
+        Publisher.logger.error("[#{id}] -  Published message not confirmed")
+      end
+    end
   end
 
-  def start_supervisor
-    @config = Celluloid::Supervision::Configuration.define(
-      [
-        { type: TransactionStore, as: :transaction_store },
-        { type: Publisher, as: :publisher }
-      ]
-    )
-
-    sleeper = @sleeper
-    @config.injection!(:before_restart, proc do
-      Example.logger.info('Restarting in 15 seconds...')
-      sleeper.start(15)
-    end)
-    @config.deploy
-  end
+  # Application
+  class Application
+    def initialize
+      @sleeper = EventHub::Sleeper.new
+      @command_queue = []
+    end
+
+    def start_supervisor
+      @config = Celluloid::Supervision::Configuration.define(
+        [
+          { type: TransactionStore, as: :transaction_store },
+          { type: Worker, as: :worker }
+        ]
+      )
 
-  def start
-    Example.logger.info 'Publisher has been started'
+      sleeper = @sleeper
+      @config.injection!(:before_restart, proc do
+        Publisher.logger.info('Restarting in 15 seconds...')
+        sleeper.start(15)
+      end)
+      @config.deploy
+    end
 
-    setup_signal_handler
-    start_supervisor
-    main_event_loop
+    def start
+      Publisher.logger.info 'Publisher has been started'
 
-    Example.logger.info 'Publisher has been stopped'
-  end
+      setup_signal_handler
+      start_supervisor
+      main_event_loop
 
-  private
-
-  def main_event_loop
-    loop do
-      command = @command_queue.pop
-      case
-        when SIGNALS_FOR_TERMINATION.include?(command)
-          @sleeper.stop
-          break
-        else
-          sleep 0.5
-      end
+      Publisher.logger.info 'Publisher has been stopped'
     end
 
-    Celluloid.shutdown
-    # make sure all actors are gone
-    while Celluloid.running?
-      sleep 0.1
+    private
+
+    def main_event_loop
+      loop do
+        command = @command_queue.pop
+        case
+          when SIGNALS_FOR_TERMINATION.include?(command)
+            @sleeper.stop
+            break
+          else
+            sleep 0.5
+        end
+      end
+
+      Celluloid.shutdown
+      # make sure all actors are gone
+      while Celluloid.running?
+        sleep 0.1
+      end
     end
-  end
 
-  def setup_signal_handler
-    # have a re-entrant signal handler by just using a simple array
-    # https://www.sitepoint.com/the-self-pipe-trick-explained/
-    ALL_SIGNALS.each do |signal|
-      Signal.trap(signal) { @command_queue << signal }
+    def setup_signal_handler
+      # have a re-entrant signal handler by just using a simple array
+      # https://www.sitepoint.com/the-self-pipe-trick-explained/
+      ALL_SIGNALS.each do |signal|
+        Signal.trap(signal) { @command_queue << signal }
+      end
     end
   end
 end
 
-Application.new.start
+Publisher::Application.new.start

data/example/receiver.rb

@@ -14,7 +14,7 @@ module EventHub
         EventHub.logger.error("[#{id}] - Unable to delete File: #{error}")
       end
 
-      []
+      nil
     end
   end
 end

data/example/router.rb

@@ -8,7 +8,7 @@ module EventHub
       EventHub.logger.info("Received: [#{id}]")
       publish(message: message.to_json, exchange_name: 'example.inbound')
       EventHub.logger.info("Returned: [#{id}]")
-      []
+      nil
     end
   end
 end

data/lib/eventhub/actor_heartbeat.rb

@@ -26,7 +26,7 @@ module EventHub
     end
 
     def cleanup
-      EventHub.logger.info('Heartbeat is cleanig up...')
+      EventHub.logger.info('Heartbeat is cleaning up...')
       publish(heartbeat(action: 'stopped'))
       EventHub.logger.info('Heartbeat has sent a [stopped] beat')
     end

data/lib/eventhub/actor_listener.rb

@@ -113,7 +113,7 @@ module EventHub
     end
 
     def cleanup
-      EventHub.logger.info('Listener is cleanig up...')
+      EventHub.logger.info('Listener is cleaning up...')
       # close all open connections
       return unless @connections
       @connections.values.each do |connection|

data/lib/eventhub/actor_publisher.rb

@@ -39,7 +39,7 @@ module EventHub
     end
 
     def cleanup
-      EventHub.logger.info('Publisher is cleanig up...')
+      EventHub.logger.info('Publisher is cleaning up...')
       @connection.close if @connection
     end
   end

data/lib/eventhub/actor_watchdog.rb

@@ -20,7 +20,7 @@ module EventHub
     end
 
     def cleanup
-      EventHub.logger.info('Watchdog is cleanig up...')
+      EventHub.logger.info('Watchdog is cleaning up...')
     end
 
     private

data/lib/eventhub/consumer.rb

@@ -1,6 +1,6 @@
 # EventHub module
 module EventHub
-  # Heartbeat class
+  # Consumer class
   class Consumer < Bunny::Consumer
     def handle_cancellation(_)
       EventHub.logger.error("Consumer reports cancellation")

data/lib/eventhub/version.rb

@@ -1,3 +1,3 @@
 module EventHub
-  VERSION = '1.1.0'.freeze
+  VERSION = '1.2.0'.freeze
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: eventhub-processor2
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.0
 platform: ruby
 authors:
 - Steiner, Thomas