eventhub-processor 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 1d655c3ad766e679c042f39ba0b9959e9607fa11
+  data.tar.gz: f0cf376227b402a5491ac3bc45ae10b4625aad89
+SHA512:
+  metadata.gz: b71cca2fbc7db9c483bd4cb689fb07efc61ca6409a21958c36a629198c5b1056ef3d26786ebb8edaa6aaeed3e2255a95e57449cf0af0e7f6a5269dd02da11f37
+  data.tar.gz: cd85de1eee523df747f5f198e1c55f9b7fa8c655cb190e7f73e6fd175454b1305b5f62cb8efc8c619262d61d8717dfb5480cde9bcd0d7fb7581bc3e903ee6283

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 thomis
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/lib/eventhub-processor.rb

@@ -0,0 +1,22 @@
+require 'amqp'
+require 'rest-client'
+require 'json'
+require 'singleton'
+
+require_relative 'eventhub/version'
+require_relative 'eventhub/multi_logger'
+
+require_relative 'eventhub/configuration'
+require_relative 'eventhub/hash'
+require_relative 'eventhub/processor'
+require_relative 'eventhub/message'
+
+module EventHub
+  def self.logger
+  	unless @logger
+  		a = Logger.new(STDOUT)
+  		@logger = MultiLogger.new(a)
+  	end
+  	@logger
+  end
+end

data/lib/eventhub/configuration.rb

@@ -0,0 +1,19 @@
+module EventHub
+
+	class Configuration
+		include Singleton
+
+		attr_accessor :data
+  	
+  	def initialize
+    	@data = nil
+  	end
+
+  	def load_file(input, env='development')
+    	tmp = JSON.parse( IO.read(input))
+  		@data = tmp[env]
+  	end
+
+	end
+
+end

data/lib/eventhub/hash.rb

@@ -0,0 +1,34 @@
+class ::Hash
+
+	# get value from provided key path, e.g. hash.get('a.b.c')
+  def get(key_path)
+    key_path.split('.').inject(self,:[])
+  rescue NoMethodError 
+    return nil
+  end
+
+  # set value from provided key path, e.h. hash.set('a.b.c','new value')
+  # if overwrite is false, value will be set if it was nil previously
+  def set(key_path,value,overwrite=true)
+    *key_path, last = key_path.split(".")
+    if overwrite
+      key_path.inject(self) { |h,key| h.has_key?(key) ? h[key] :  h[key]={}} [last] = value 
+    else
+      key_path.inject(self) { |h,key| h.has_key?(key) ? h[key] :  h[key]={}} [last] ||= value
+    end  
+  end
+
+  # get all keys path, { 'a' => 'value1', 'b' => { 'c' => 'value2'}}.all_keys_with_path => ['a','b.c']
+  def all_keys_with_path(parent=nil)
+  	a = []
+  	each do |k,v|
+  		if v.is_a?(Hash)
+  			a << v.all_keys_with_path([parent,k].compact.join('.'))
+  		else
+  			a << "#{[parent,k].compact.join(".")}"
+  		end
+  	end
+  	a.flatten
+  end
+
+end

data/lib/eventhub/message.rb

@@ -0,0 +1,106 @@
+module EventHub
+  
+  class Message
+
+    VERSION = '1.0.0'
+
+    # Headers that are required (value can be nil) in order to pass valid?
+    REQUIRED_HEADERS = [
+      'message_id',
+      'version',
+      'created_at',
+      'origin.module_id',
+      'origin.type',
+      'origin.site_id',
+      'process.name',
+      'process.step_position',
+      'process.execution_id',
+      'status.retried_count',
+      'status.code',
+      'status.message'
+    ]
+
+    attr_accessor :header, :body, :raw, :vhost, :routing_key
+
+    # Build accessors for all required headers
+    REQUIRED_HEADERS.each do |header|
+      name = header.gsub(/\./,"_")
+
+      define_method(name) do
+        self.header.get(header)
+      end
+
+      define_method("#{name}=") do |value|
+        self.header.set(header,value)
+      end
+    end
+
+    def self.from_json(json)
+      data = JSON.parse(json)
+      Eventhub::Message.new(data.get('header'), data.get('body'),json)
+    rescue => e
+      Eventhub::Message.new({ "status" =>  { "code" => STATUS_INVALID, "message" => "JSON parse error: #{e}" }} ,{},json)
+    end
+
+    # process_step_position should be
+    def initialize(header, body,raw=nil)
+
+      @header = header || {}
+      @body   = body || {}
+      @raw    = raw
+
+      # set message defaults, that we have required headers
+      now = Time.now
+      @header.set('message_id',UUIDTools::UUID.timestamp_create.to_s,false)
+      @header.set('version',VERSION,false)
+      @header.set('created_at',now.utc.strftime("%Y-%m-%dT%H:%M:%S.#{now.usec/1000}Z"),false)
+
+      @header.set('process.name',nil,false)
+      @header.set('process.execution_id',UUIDTools::UUID.timestamp_create.to_s,false)
+      @header.set('process.step_position',0,false)
+
+      @header.set('status.retried_count',0,false)
+      @header.set('status.code',STATUS_INITIAL,false)
+      @header.set('status.message',nil,false)
+
+    end
+
+    def valid?
+      REQUIRED_HEADERS.all? { |key| @header.all_keys_with_path.include?(key) }
+    end
+
+    def success?
+      self.status_code == STATUS_SUCCESS
+    end
+
+    def retry?
+      !success?
+    end
+
+    def initial?
+      self.status_code == STATUS_INITIAL
+    end
+
+    def retried?
+      self.status_code == STATUS_RETRIED
+    end
+
+    def to_json
+      {'header' => self.header, 'body' => self.body}.to_json
+    end
+
+    def to_s
+      "Message: message_id [#{self.message_id}], status.code [#{status_code}], status.message [#{status_message}], status.retried_count [#{status_retried_count}] "
+    end
+
+    def copy(args={})
+      copied_header = self.header.dup
+      copied_body   = self.body.dup
+
+      args.each { |key,value| copied_header.set(key,value) } if args.is_a?(Hash)
+      Eventhub::Message.new(copied_header, copied_body)
+    end  
+
+  end
+
+end

data/lib/eventhub/multi_logger.rb

@@ -0,0 +1,61 @@
+require 'logger'
+
+# format adaptation
+class Logger
+  class Formatter
+    def call(severity, time, progname, msg)
+      time_in_string = "#{time.strftime("%Y-%m-%d %H:%M:%S")}.#{"%04d" % (time.usec/100)}"
+      [time_in_string,Process.pid,severity,msg].join("\t") + "\n"
+    end
+  end
+  
+end
+
+
+module EventHub
+
+  class MultiLogger
+    def initialize(*targets)
+        @targets = targets
+    end
+
+    def save_detailed_error(feedback,message=nil)
+    	time = Time.now
+      stamp = "#{time.strftime("%Y%m%d_%H%M%S")}_#{"%03d" % (time.usec/1000)}"
+      filename = "#{stamp}.log"
+
+    	FileUtils.makedirs("exceptions")
+
+      File.open("exceptions/#{filename}","w") do |output|
+        output.write("#{feedback}\n\n")
+        output.write("Exception: #{feedback.class.to_s}\n\n")
+        output.write("Call Stack:\n")
+        feedback.backtrace.each do |line|
+          output.write("#{line}\n")
+        end
+      end
+
+      # save message if provided
+      if message
+  	    File.open("exceptions/#{stamp}.msg.raw","wb") do |output|
+  	    	output.write(message)
+  	    end
+    	end	
+
+      return stamp
+    end
+
+    %w(log debug info warn error).each do |m|
+      define_method(m) do |*args|
+          @targets.map { |t| t.send(m, *args) }
+      end
+    end
+
+  end
+
+end
+
+
+
+  
+

data/lib/eventhub/processor.rb

@@ -0,0 +1,139 @@
+module EventHub
+	class Processor
+
+		def hostname
+			configuration.get('server.hostname') || 'localhost'
+		end
+
+		def user
+			configuration.get('server.user') || 'admin'
+		end
+
+		def password
+			configuration.get('server.password') || 'admin'
+		end
+
+		def management_port
+			configuration.get('server.management_port') || 15672
+		end
+
+		def queue_name
+			configuration.get('processor.queue') || 'inbound'
+		end
+
+		def vhost
+			configuration.get('server.vhost') || nil
+		end
+
+		def watchdog_cycle
+			configuration.get('processor.watchdog_cycle') || 5
+		end
+
+		def configuration
+			EventHub::Configuration.instance.data
+		end
+
+
+
+		def start
+			restart = true
+			while restart
+
+				begin 
+					AMQP.start(configuration.get('server')) do |connection, open_ok|
+						
+						# deal with tcp connection issues
+						connection.on_tcp_connection_loss do |conn, settings|
+				  		EventHub.logger.warn("Processor lost tcp connection. Trying to restart in 5 seconds...")
+				  		sleep 5
+				    	EventMachine.stop
+				  	end
+
+						# create channel
+						channel = AMQP::Channel.new(connection)
+				  	channel.auto_recovery = true
+				  
+				  	# connect to queue
+				  	queue = channel.queue(configuration.get('processor.queue'), durable: true, auto_delete: false)
+
+				  	# subscribe to queue
+					  queue.subscribe do |metadata, payload|
+					  	handle_heartbeat(payload)
+					  	handle_message(metadata,payload)
+					  end
+
+						# Features to stop main event loop
+						stop_main_loop = Proc.new {
+					    connection.disconnect { 
+					    	EventHub.logger.info("Processor is stopping main event loop")
+					    	EventMachine.stop
+					    	restart = false
+					    }
+				  	}
+
+					  Signal.trap "TERM", stop_main_loop
+					  Signal.trap "INT",  stop_main_loop
+
+					  EventMachine.add_timer(self.watchdog_cycle) { watchdog }
+
+					  EventHub.logger.info("Processor is listening to queue [#{[configuration.get('server.vhost'),configuration.get('processor.queue')].compact.join(".")}]")
+					end
+				rescue => e
+					EventHub.logger.error("Unexpected exception: #{e}. Trying to restart in 5 seconds...")
+					sleep 5
+				end
+
+			end # while
+
+			EventHub.logger.info("Processor has been stopped")
+		end
+
+		def handle_message(metadata,payload)
+			raise "Please implement method in derived class"
+		end
+
+		def handle_heartbeat(message)
+			# sends a standard message back to dispatcher
+		end
+
+		def watchdog
+			begin
+				response = RestClient.get "http://#{self.user}:#{self.password}@#{hostname}:#{management_port}/api/queues/#{self.vhost}/#{self.queue_name}/bindings", { :content_type => :json}
+  			data = JSON.parse(response.body)
+  	
+  			if response.code != 200
+  				EventHub.logger.warn("Watchdog: Server did not answered properly. Trying to restart in 5 seconds...")
+  				sleep 5
+  				EventMachine.stop
+  			elsif data.size == 0
+  				EventHub.logger.warn("Watchdog: Something is wrong with the vhost, queue, and/or bindings. Trying to restart in 5 seconds...")
+  				sleep 5
+  				EventMachine.stop
+  			else
+  				# Watchdog is happy :-)
+  			end
+
+			rescue => e
+				EventHub.logger.error("Watchdog: Unexpected exception: #{e}. Trying to restart in 5 seconds...")
+				sleep 5
+				EventMachine.stop
+			end
+
+			# place next time
+			EventMachine.add_timer(watchdog_cycle) { watchdog }
+		end
+
+		def sent_to_dispatcher(payload)
+			# send_connection = AMQP.connect({hostname: self.hostname, user: self.user, password: self.password, vhost: "event_hub"})
+  		
+  	# 	send_channel  = AMQP::Channel.new(send_connection)
+  	# 	send_exchange = send_channel.direct("")
+
+  	# 	send_exchange.publish payload, :routing_key => 'inbound'
+		
+			# send_channel.close
+			# send_conncetion.close
+		end
+
+	end
+end

data/lib/eventhub/version.rb

@@ -0,0 +1,3 @@
+module EventHub
+  VERSION = "0.0.1"
+end

metadata

@@ -0,0 +1,109 @@
+--- !ruby/object:Gem::Specification
+name: eventhub-processor
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Thomas Steiner
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-04-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: amqp
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Gem to build Event Hub processors
+email:
+- thomas.steiner@ikey.ch
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/eventhub/configuration.rb
+- lib/eventhub/hash.rb
+- lib/eventhub/message.rb
+- lib/eventhub/multi_logger.rb
+- lib/eventhub/processor.rb
+- lib/eventhub/version.rb
+- lib/eventhub-processor.rb
+- LICENSE.txt
+- Rakefile
+homepage: http://github.com/thomis/eventhub-processor
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Gem to build Event Hub processors
+test_files: []