eventhub-processor2 1.0.0

data/lib/eventhub/message.rb

@@ -0,0 +1,158 @@
+# EventHub module
+module EventHub
+  # Message class
+  class Message
+    include Helper
+
+    VERSION = '1.0.0'.freeze
+
+    # Headers that are required (value can be nil) in order to pass valid?
+    REQUIRED_HEADERS = [
+      'message_id',
+      'version',
+      'created_at',
+      'origin.module_id',
+      'origin.type',
+      'origin.site_id',
+      'process.name',
+      'process.step_position',
+      'process.execution_id',
+      'status.retried_count',
+      'status.code',
+      'status.message'
+    ].freeze
+
+    attr_accessor :header, :body, :raw, :vhost, :routing_key
+
+    # Build accessors for all required headers
+    REQUIRED_HEADERS.each do |header|
+      name = header.tr('.', '_')
+
+      define_method(name) do
+        self.header.get(header)
+      end
+
+      define_method("#{name}=") do |value|
+        self.header.set(header, value)
+      end
+    end
+
+    def self.from_json(raw)
+      data = JSON.parse(raw)
+      Message.new(data.get('header'), data.get('body'), raw)
+    rescue => e
+      Message.new(
+        {
+          'status' =>
+          {
+            'code' => STATUS_INVALID,
+            'message' => "JSON parse error: #{e}"
+          }
+        },
+        {
+          'original_message_base64_encoded' => Base64.encode64(raw)
+        },
+        raw
+      )
+    end
+
+    def initialize(header = nil, body = nil, raw = nil)
+      @header = header || {}
+      @body   = body || {}
+      @raw    = raw
+
+      # set message defaults, that we have required headers
+      @header.set('message_id', UUIDTools::UUID.timestamp_create.to_s, false)
+      @header.set('version', VERSION, false)
+      @header.set('created_at', now_stamp, false)
+
+      @header.set('origin.module_id', 'undefined', false)
+      @header.set('origin.type', 'undefined', false)
+      @header.set('origin.site_id', 'undefined', false)
+
+      @header.set('process.name', 'undefined', false)
+      @header.set('process.execution_id',
+                  UUIDTools::UUID.timestamp_create.to_s, false)
+      @header.set('process.step_position', 0, false)
+
+      @header.set('status.retried_count', 0, false)
+      @header.set('status.code', STATUS_INITIAL, false)
+      @header.set('status.message', '', false)
+    end
+
+    def valid?
+      # check for existence and defined value
+      REQUIRED_HEADERS.all? do |key|
+        @header.all_keys_with_path.include?(key) &&
+          !send(key.tr('.', '_').to_sym).nil?
+      end
+    end
+
+    def success?
+      status_code == STATUS_SUCCESS
+    end
+
+    def retry?
+      status_code == STATUS_RETRY
+    end
+
+    def initial?
+      status_code == STATUS_INITIAL
+    end
+
+    def retry_pending?
+      status_code == STATUS_RETRY_PENDING
+    end
+
+    def invalid?
+      status_code == STATUS_INVALID
+    end
+
+    def schedule?
+      status_code == STATUS_SCHEDULE
+    end
+
+    def schedule_retry?
+      status_code == STATUS_SCHEDULE_RETRY
+    end
+
+    def schedule_pending?
+      status_code == STATUS_SCHEDULE_PENDING
+    end
+
+    def to_json
+      { 'header' => header, 'body' => body }.to_json
+    end
+
+    def to_s
+      'Msg: process '\
+        "[#{process_name}, #{process_step_position}, #{process_execution_id}]"\
+        ", status [#{status_code},#{status_message},#{status_retried_count}]"
+    end
+
+    # copies the message and set's provided status code (default: success),
+    # actual stamp, and a new message id
+    def copy(status_code = STATUS_SUCCESS)
+      # use Marshal dump and load to make a deep object copy
+      copied_header = Marshal.load(Marshal.dump(header))
+      copied_body   = Marshal.load(Marshal.dump(body))
+
+      copied_header.set('message_id', UUIDTools::UUID.timestamp_create.to_s)
+      copied_header.set('created_at', now_stamp)
+      copied_header.set('status.code', status_code)
+
+      Message.new(copied_header, copied_body)
+    end
+
+    def append_to_execution_history(processor_name)
+      header.set('execution_history', []) unless \
+        header.get('execution_history')
+      header.get('execution_history') << \
+        { 'processor' => processor_name, 'timestamp' => now_stamp }
+    end
+
+    def self.translate_status_code(code)
+      STATUS_CODE_TRANSLATION[code]
+    end
+  end
+end

data/lib/eventhub/processor2.rb

@@ -0,0 +1,121 @@
+# EventHub module
+module EventHub
+  # Processor2 class
+  class Processor2
+    include Helper
+
+    SIGNALS_FOR_TERMINATION = [:INT, :TERM, :QUIT]
+    SIGNALS_FOR_RELOAD_CONFIG = [:HUP]
+    ALL_SIGNALS = SIGNALS_FOR_TERMINATION + SIGNALS_FOR_RELOAD_CONFIG
+
+    attr_reader :started_at, :statistics
+
+    def initialize(args = {})
+      # Set processor name
+      EventHub::Configuration.name = args[:name] ||
+                                     get_name_from_class(self)
+
+      # Parse comand line options
+      EventHub::Configuration.parse_options
+
+      # Load configuration file
+      EventHub::Configuration.load!(args)
+
+      @command_queue = []
+
+      @started_at = Time.now
+      @statistics = EventHub::Statistics.new
+    end
+
+    def start
+      EventHub.logger.info("#{Configuration.name} (#{version}): has been started")
+
+      before_start
+      main_event_loop
+      after_stop
+
+      EventHub.logger.info("#{Configuration.name} (#{version}): has been stopped")
+    rescue => ex
+      EventHub.logger.error("Unexpected error in Processor2.start: #{ex}")
+    end
+
+    def stop
+      # used by rspec
+      @command_queue << :TERM
+    end
+
+    def version
+      EventHub::VERSION
+    end
+
+    # get message as EventHub::Message class instance
+    # args contain :queue_name, :content_type, :priority, :delivery_tag
+    def handle_message(_message, _args = {})
+      raise 'need to be implemented in derived class'
+    end
+
+    # pass message: '{ "header": ... , "body": { .. }}'
+    # and optionally exchange_name: 'your exchange name'
+    def publish(args = {})
+      Celluloid::Actor[:actor_listener].publish(args)
+    end
+
+    def before_start
+      # can be implemented in derived class
+    end
+
+    def after_stop
+      # can be implemented in derived class
+    end
+
+    private
+
+    def setup_signal_handler
+      # have a re-entrant signal handler by just using a simple array
+      # https://www.sitepoint.com/the-self-pipe-trick-explained/
+      ALL_SIGNALS.each do |signal|
+        Signal.trap(signal) { @command_queue << signal }
+      end
+    end
+
+    def start_supervisor
+      @config = Celluloid::Supervision::Configuration.define([
+        {type: ActorHeartbeat, as: :actor_heartbeat, args: [ self ]},
+        {type: ActorListener, as: :actor_listener, args: [ self ]}
+      ])
+
+      @config.injection!(:before_restart, proc do
+        EventHub.logger.info('Restarting in 10 seconds...')
+        sleep 10
+      end )
+
+      @config.deploy
+    end
+
+    def main_event_loop
+      setup_signal_handler
+      start_supervisor
+
+      loop do
+        command = @command_queue.pop
+        case
+          when SIGNALS_FOR_TERMINATION.include?(command)
+            EventHub.logger.info("Command [#{command}] received")
+            break
+          when SIGNALS_FOR_RELOAD_CONFIG.include?(command)
+            EventHub::Configuration.load!
+            EventHub.logger.info('Configuration file reloaded')
+            Celluloid::Actor[:actor_listener].async.restart
+          else
+            sleep 0.5
+        end
+      end
+
+      Celluloid.shutdown
+      # make sure all actors are gone
+      while Celluloid.running?
+        sleep 0.1
+      end
+    end
+  end
+end

data/lib/eventhub/statistics.rb

@@ -0,0 +1,53 @@
+class EventHub::Statistics
+  attr_reader :messages_successful, :messages_unsuccessful, :messages_average_size, :messages_average_process_time
+
+  def initialize
+    @messages_successful = 0
+    @messages_unsuccessful = 0
+    @messages_average_size = 0
+    @messages_average_process_time = 0
+    @messages_total_process_time = 0
+    @mutex = Mutex.new
+  end
+
+
+  def measure(size, &block)
+    begin
+      start = Time.now
+      yield
+      success(Time.now - start, size)
+    rescue
+      failure
+      raise
+    end
+  end
+
+  def success(process_time, size)
+    @mutex.lock
+    @messages_total_process_time += process_time
+    @messages_average_process_time = (messages_total_process_time + process_time) / (messages_successful + 1).to_f
+    @messages_average_size = (messages_total_size + size) / (messages_successful + 1).to_f
+    @messages_successful += 1
+  ensure
+    @mutex.unlock
+  end
+
+  def failure
+    @mutex.lock
+    @messages_unsuccessful += 1
+  ensure
+    @mutex.unlock
+  end
+
+  def messages_total
+    messages_unsuccessful + messages_successful
+  end
+
+  def messages_total_process_time
+    messages_average_process_time * messages_successful
+  end
+
+  def messages_total_size
+    messages_average_size * messages_successful
+  end
+end

data/lib/eventhub/version.rb

@@ -0,0 +1,3 @@
+module EventHub
+  VERSION = '1.0.0'.freeze
+end

data/lib/eventhub.rb

@@ -0,0 +1,25 @@
+require 'uuidtools'
+require 'json'
+require 'base64'
+
+require 'eventhub/components'
+require 'logstash-logger'
+require 'bunny'
+require 'celluloid/current'
+
+require_relative 'eventhub/version'
+require_relative 'eventhub/constant'
+require_relative 'eventhub/logger'
+require_relative 'eventhub/helper'
+require_relative 'eventhub/hash_extensions'
+require_relative 'eventhub/configuration'
+require_relative 'eventhub/message'
+require_relative 'eventhub/statistics'
+require_relative 'eventhub/consumer'
+require_relative 'eventhub/actor_heartbeat'
+require_relative 'eventhub/actor_watchdog'
+require_relative 'eventhub/actor_listener'
+require_relative 'eventhub/processor2'
+
+Celluloid.logger = nil
+Celluloid.exception_handler { |ex| EventHub.logger.info "Exception occured: #{ex}" }

metadata

@@ -0,0 +1,190 @@
+--- !ruby/object:Gem::Specification
+name: eventhub-processor2
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Steiner, Thomas
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-12-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: celluloid
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.17'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.17'
+- !ruby/object:Gem::Dependency
+  name: bunny
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+- !ruby/object:Gem::Dependency
+  name: eventhub-components
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: uuidtools
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.2'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.7'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.15'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.15'
+description: Next generation gem to build ruby based eventhub processor
+email:
+- thomas.steiner@ikey.ch
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- docker/Dockerfile
+- docker/README.md
+- docker/definitions.json
+- docker/rabbitmq.config
+- eventhub-processor2.gemspec
+- example/README.md
+- example/config/example.json
+- example/crasher.rb
+- example/example.rb
+- example/publisher.rb
+- lib/eventhub.rb
+- lib/eventhub/actor_heartbeat.rb
+- lib/eventhub/actor_listener.rb
+- lib/eventhub/actor_watchdog.rb
+- lib/eventhub/configuration.rb
+- lib/eventhub/constant.rb
+- lib/eventhub/consumer.rb
+- lib/eventhub/hash_extensions.rb
+- lib/eventhub/helper.rb
+- lib/eventhub/logger.rb
+- lib/eventhub/message.rb
+- lib/eventhub/processor2.rb
+- lib/eventhub/statistics.rb
+- lib/eventhub/version.rb
+homepage: https://github.com/thomis/eventhub-processor2
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.12
+signing_key: 
+specification_version: 4
+summary: Next generation gem to build ruby based eventhub processor
+test_files: []