eventhub-processor 0.3.1 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/eventhub-processor.rb +37 -37
- data/lib/eventhub/argument_parser.rb +34 -34
- data/lib/eventhub/base_exception.rb +1 -1
- data/lib/eventhub/configuration.rb +25 -25
- data/lib/eventhub/constant.rb +15 -15
- data/lib/eventhub/hash_extensions.rb +48 -48
- data/lib/eventhub/heartbeat.rb +77 -77
- data/lib/eventhub/helper.rb +49 -49
- data/lib/eventhub/message.rb +138 -138
- data/lib/eventhub/message_processor.rb +31 -31
- data/lib/eventhub/no_deadletter_exception.rb +1 -1
- data/lib/eventhub/processor.rb +269 -268
- data/lib/eventhub/statistics.rb +47 -47
- data/lib/eventhub/version.rb +3 -3
- metadata +17 -5
- data/lib/eventhub/multi_logger.rb +0 -89
- data/lib/eventhub/pidfile.rb +0 -21
- data/lib/eventhub/test.rb +0 -10
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7aa5e7a8e2f31b9683c5ff655c76c1c46bc75372
|
4
|
+
data.tar.gz: 717a76158ee4c97ba9b5595a8d21814bd36d0102
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ba810bf60a874be8e0611d2156e17a90e67a12dbfe68258d17b11ddcfdeb490fd237b8426322328d46b6bea5d818f5078c29bb5b944e3534d1f1b964dea72104
|
7
|
+
data.tar.gz: 335f306fd2c7300967fd702dace1239eca7a07b4030ee1b0be50b3f723c627704c15027e944d7c159bddac96b0e4cf440cf9574b8637f901c4f1a8a11fc81cb9
|
data/lib/eventhub-processor.rb
CHANGED
@@ -1,37 +1,37 @@
|
|
1
|
-
require 'amqp'
|
2
|
-
require 'rest-client'
|
3
|
-
require 'json'
|
4
|
-
require 'singleton'
|
5
|
-
require 'uuidtools'
|
6
|
-
require 'base64'
|
7
|
-
require 'socket'
|
8
|
-
require 'ostruct'
|
9
|
-
require 'optparse'
|
10
|
-
|
11
|
-
require_relative 'eventhub/argument_parser'
|
12
|
-
require_relative 'eventhub/version'
|
13
|
-
require_relative 'eventhub/constant'
|
14
|
-
require_relative 'eventhub/helper'
|
15
|
-
require_relative 'eventhub/multi_logger'
|
16
|
-
|
17
|
-
require_relative 'eventhub/configuration'
|
18
|
-
require_relative 'eventhub/hash_extensions'
|
19
|
-
require_relative 'eventhub/statistics'
|
20
|
-
require_relative 'eventhub/heartbeat'
|
21
|
-
require_relative 'eventhub/pidfile'
|
22
|
-
require_relative 'eventhub/processor'
|
23
|
-
require_relative 'eventhub/message_processor'
|
24
|
-
require_relative 'eventhub/message'
|
25
|
-
|
26
|
-
require_relative 'eventhub/base_exception'
|
27
|
-
require_relative 'eventhub/no_deadletter_exception'
|
28
|
-
|
29
|
-
module EventHub
|
30
|
-
def self.logger
|
31
|
-
unless @logger
|
32
|
-
@logger = MultiLogger.new
|
33
|
-
@logger.add_device(Logger.new(STDOUT))
|
34
|
-
end
|
35
|
-
@logger
|
36
|
-
end
|
37
|
-
end
|
1
|
+
require 'amqp'
|
2
|
+
require 'rest-client'
|
3
|
+
require 'json'
|
4
|
+
require 'singleton'
|
5
|
+
require 'uuidtools'
|
6
|
+
require 'base64'
|
7
|
+
require 'socket'
|
8
|
+
require 'ostruct'
|
9
|
+
require 'optparse'
|
10
|
+
|
11
|
+
require_relative 'eventhub/argument_parser'
|
12
|
+
require_relative 'eventhub/version'
|
13
|
+
require_relative 'eventhub/constant'
|
14
|
+
require_relative 'eventhub/helper'
|
15
|
+
require_relative 'eventhub/multi_logger'
|
16
|
+
|
17
|
+
require_relative 'eventhub/configuration'
|
18
|
+
require_relative 'eventhub/hash_extensions'
|
19
|
+
require_relative 'eventhub/statistics'
|
20
|
+
require_relative 'eventhub/heartbeat'
|
21
|
+
require_relative 'eventhub/pidfile'
|
22
|
+
require_relative 'eventhub/processor'
|
23
|
+
require_relative 'eventhub/message_processor'
|
24
|
+
require_relative 'eventhub/message'
|
25
|
+
|
26
|
+
require_relative 'eventhub/base_exception'
|
27
|
+
require_relative 'eventhub/no_deadletter_exception'
|
28
|
+
|
29
|
+
module EventHub
|
30
|
+
def self.logger
|
31
|
+
unless @logger
|
32
|
+
@logger = ::EventHub::Components::MultiLogger.new
|
33
|
+
@logger.add_device(Logger.new(STDOUT))
|
34
|
+
end
|
35
|
+
@logger
|
36
|
+
end
|
37
|
+
end
|
@@ -1,35 +1,35 @@
|
|
1
|
-
module EventHub
|
2
|
-
|
3
|
-
class ArgumentParser
|
4
|
-
|
5
|
-
def self.parse(args)
|
6
|
-
# The options specified on the command line will be collected in *options*.
|
7
|
-
# We set default values here.
|
8
|
-
options = OpenStruct.new
|
9
|
-
options.environment = 'development'
|
10
|
-
options.detached = false
|
11
|
-
|
12
|
-
opt_parser = OptionParser.new do |opts|
|
13
|
-
opts.banner = "Usage: #{args[0]}.rb [options]"
|
14
|
-
yield(opts, options) if block_given? # allow to add more options
|
15
|
-
|
16
|
-
opts.on("-e", "--environment ENVIRONMENT","Environment the processor is running") do |environment|
|
17
|
-
options.environment = environment
|
18
|
-
end
|
19
|
-
|
20
|
-
opts.on("-d", "--detached", "Run processor detached as a daemon") do |v|
|
21
|
-
options.detached = v
|
22
|
-
end
|
23
|
-
|
24
|
-
opts.on_tail("-h", "--help", "Show this message") do
|
25
|
-
puts opts
|
26
|
-
exit
|
27
|
-
end
|
28
|
-
end
|
29
|
-
|
30
|
-
opt_parser.parse!(args)
|
31
|
-
options
|
32
|
-
end
|
33
|
-
end
|
34
|
-
|
1
|
+
module EventHub
|
2
|
+
|
3
|
+
class ArgumentParser
|
4
|
+
|
5
|
+
def self.parse(args)
|
6
|
+
# The options specified on the command line will be collected in *options*.
|
7
|
+
# We set default values here.
|
8
|
+
options = OpenStruct.new
|
9
|
+
options.environment = 'development'
|
10
|
+
options.detached = false
|
11
|
+
|
12
|
+
opt_parser = OptionParser.new do |opts|
|
13
|
+
opts.banner = "Usage: #{args[0]}.rb [options]"
|
14
|
+
yield(opts, options) if block_given? # allow to add more options
|
15
|
+
|
16
|
+
opts.on("-e", "--environment ENVIRONMENT","Environment the processor is running") do |environment|
|
17
|
+
options.environment = environment
|
18
|
+
end
|
19
|
+
|
20
|
+
opts.on("-d", "--detached", "Run processor detached as a daemon") do |v|
|
21
|
+
options.detached = v
|
22
|
+
end
|
23
|
+
|
24
|
+
opts.on_tail("-h", "--help", "Show this message") do
|
25
|
+
puts opts
|
26
|
+
exit
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
opt_parser.parse!(args)
|
31
|
+
options
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
35
|
end
|
@@ -1,2 +1,2 @@
|
|
1
|
-
class EventHub::BaseException < RuntimeError
|
1
|
+
class EventHub::BaseException < RuntimeError
|
2
2
|
end
|
@@ -1,26 +1,26 @@
|
|
1
|
-
module EventHub
|
2
|
-
|
3
|
-
class Configuration
|
4
|
-
include Singleton
|
5
|
-
include Helper
|
6
|
-
|
7
|
-
attr_accessor :data, :folder, :environment
|
8
|
-
|
9
|
-
def initialize
|
10
|
-
@data = nil
|
11
|
-
@environment = 'development'
|
12
|
-
end
|
13
|
-
|
14
|
-
def load_file(input, env = 'development')
|
15
|
-
json = JSON.parse(IO.read(input))
|
16
|
-
@data = json[env]
|
17
|
-
@environment = env
|
18
|
-
true
|
19
|
-
rescue => e
|
20
|
-
EventHub.logger.info("Unexpected exception while loading configuration [#{input}]: #{format_string(e.message)}")
|
21
|
-
false
|
22
|
-
end
|
23
|
-
|
24
|
-
end
|
25
|
-
|
1
|
+
module EventHub
|
2
|
+
|
3
|
+
class Configuration
|
4
|
+
include Singleton
|
5
|
+
include Helper
|
6
|
+
|
7
|
+
attr_accessor :data, :folder, :environment
|
8
|
+
|
9
|
+
def initialize
|
10
|
+
@data = nil
|
11
|
+
@environment = 'development'
|
12
|
+
end
|
13
|
+
|
14
|
+
def load_file(input, env = 'development')
|
15
|
+
json = JSON.parse(IO.read(input))
|
16
|
+
@data = json[env]
|
17
|
+
@environment = env
|
18
|
+
true
|
19
|
+
rescue => e
|
20
|
+
EventHub.logger.info("Unexpected exception while loading configuration [#{input}]: #{format_string(e.message)}")
|
21
|
+
false
|
22
|
+
end
|
23
|
+
|
24
|
+
end
|
25
|
+
|
26
26
|
end
|
data/lib/eventhub/constant.rb
CHANGED
@@ -1,16 +1,16 @@
|
|
1
|
-
module EventHub
|
2
|
-
|
3
|
-
EH_X_INBOUND = 'event_hub.inbound'
|
4
|
-
|
5
|
-
STATUS_INITIAL = 0 # To be set when dispatcher needs to dispatch to first process step.
|
6
|
-
STATUS_SUCCESS = 200 # To be set to indicate successful processed message. Dispatcher will routes message to the next step.
|
7
|
-
STATUS_RETRY = 300 # To be set to trigger retry cycle controlled by the dispatcher
|
8
|
-
STATUS_RETRY_PENDING = 301 # Set and used by the dispatcher only.
|
9
|
-
# Set before putting the message into a retry queue.
|
10
|
-
# Once message has been retried it will sent do the same step with status.code = STATUS_SUCCESS
|
11
|
-
STATUS_INVALID = 400 # To be set to indicate invalid message (not json, invalid Event Hub Message).
|
12
|
-
# Dispatcher will publish message to the invalid queue.
|
13
|
-
STATUS_DEADLETTER = 500 # To be set by dispatcher, processor or channel adapters to indicate
|
14
|
-
# that message needs to be dead-lettered. Rejected messages could miss the
|
15
|
-
# status.code = STATUS_DEADLETTER due to the RabbitMQ deadletter exchange mechanism.
|
1
|
+
module EventHub
|
2
|
+
|
3
|
+
EH_X_INBOUND = 'event_hub.inbound'
|
4
|
+
|
5
|
+
STATUS_INITIAL = 0 # To be set when dispatcher needs to dispatch to first process step.
|
6
|
+
STATUS_SUCCESS = 200 # To be set to indicate successful processed message. Dispatcher will routes message to the next step.
|
7
|
+
STATUS_RETRY = 300 # To be set to trigger retry cycle controlled by the dispatcher
|
8
|
+
STATUS_RETRY_PENDING = 301 # Set and used by the dispatcher only.
|
9
|
+
# Set before putting the message into a retry queue.
|
10
|
+
# Once message has been retried it will sent do the same step with status.code = STATUS_SUCCESS
|
11
|
+
STATUS_INVALID = 400 # To be set to indicate invalid message (not json, invalid Event Hub Message).
|
12
|
+
# Dispatcher will publish message to the invalid queue.
|
13
|
+
STATUS_DEADLETTER = 500 # To be set by dispatcher, processor or channel adapters to indicate
|
14
|
+
# that message needs to be dead-lettered. Rejected messages could miss the
|
15
|
+
# status.code = STATUS_DEADLETTER due to the RabbitMQ deadletter exchange mechanism.
|
16
16
|
end
|
@@ -1,48 +1,48 @@
|
|
1
|
-
module HashExtensions
|
2
|
-
|
3
|
-
module ClassMethods
|
4
|
-
end
|
5
|
-
|
6
|
-
module InstanceMethods
|
7
|
-
# get value from provided key path, e.g. hash.get(%w(event_hub plate.queue1 retry_s))
|
8
|
-
# "a" => { "b" => { "c" => { "value"}}}
|
9
|
-
def get(arg)
|
10
|
-
path = arg.is_a?(String) ? arg.split('.') : arg
|
11
|
-
path.inject(self,:[])
|
12
|
-
rescue NoMethodError
|
13
|
-
return nil
|
14
|
-
end
|
15
|
-
|
16
|
-
# set value from provided key path, e.h. hash.set('a.b.c','new value')
|
17
|
-
# if overwrite is false, value will be set if it was nil previously
|
18
|
-
def set(arg,value,overwrite=true)
|
19
|
-
*key_path, last = arg.is_a?(String) ? arg.split(".") : arg
|
20
|
-
if overwrite
|
21
|
-
key_path.inject(self) { |h,key| h.has_key?(key) ? h[key] : h[key]={}} [last] = value
|
22
|
-
else
|
23
|
-
key_path.inject(self) { |h,key| h.has_key?(key) ? h[key] : h[key]={}} [last] ||= value
|
24
|
-
end
|
25
|
-
end
|
26
|
-
|
27
|
-
# get all keys path, { 'a' => 'value1', 'b' => { 'c' => 'value2'}}.all_keys_with_path => ['a','b.c']
|
28
|
-
def all_keys_with_path(parent=nil)
|
29
|
-
a = []
|
30
|
-
each do |k,v|
|
31
|
-
if v.is_a?(Hash)
|
32
|
-
a << v.all_keys_with_path([parent,k].compact.join('.'))
|
33
|
-
else
|
34
|
-
a << "#{[parent,k].compact.join(".")}"
|
35
|
-
end
|
36
|
-
end
|
37
|
-
a.flatten
|
38
|
-
end
|
39
|
-
|
40
|
-
end
|
41
|
-
|
42
|
-
def self.included(receiver)
|
43
|
-
receiver.extend ClassMethods
|
44
|
-
receiver.send :include, InstanceMethods
|
45
|
-
end
|
46
|
-
end
|
47
|
-
|
48
|
-
HashExtensions.included(Hash)
|
1
|
+
module HashExtensions
|
2
|
+
|
3
|
+
module ClassMethods
|
4
|
+
end
|
5
|
+
|
6
|
+
module InstanceMethods
|
7
|
+
# get value from provided key path, e.g. hash.get(%w(event_hub plate.queue1 retry_s))
|
8
|
+
# "a" => { "b" => { "c" => { "value"}}}
|
9
|
+
def get(arg)
|
10
|
+
path = arg.is_a?(String) ? arg.split('.') : arg
|
11
|
+
path.inject(self,:[])
|
12
|
+
rescue NoMethodError
|
13
|
+
return nil
|
14
|
+
end
|
15
|
+
|
16
|
+
# set value from provided key path, e.h. hash.set('a.b.c','new value')
|
17
|
+
# if overwrite is false, value will be set if it was nil previously
|
18
|
+
def set(arg,value,overwrite=true)
|
19
|
+
*key_path, last = arg.is_a?(String) ? arg.split(".") : arg
|
20
|
+
if overwrite
|
21
|
+
key_path.inject(self) { |h,key| h.has_key?(key) ? h[key] : h[key]={}} [last] = value
|
22
|
+
else
|
23
|
+
key_path.inject(self) { |h,key| h.has_key?(key) ? h[key] : h[key]={}} [last] ||= value
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
# get all keys path, { 'a' => 'value1', 'b' => { 'c' => 'value2'}}.all_keys_with_path => ['a','b.c']
|
28
|
+
def all_keys_with_path(parent=nil)
|
29
|
+
a = []
|
30
|
+
each do |k,v|
|
31
|
+
if v.is_a?(Hash)
|
32
|
+
a << v.all_keys_with_path([parent,k].compact.join('.'))
|
33
|
+
else
|
34
|
+
a << "#{[parent,k].compact.join(".")}"
|
35
|
+
end
|
36
|
+
end
|
37
|
+
a.flatten
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
41
|
+
|
42
|
+
def self.included(receiver)
|
43
|
+
receiver.extend ClassMethods
|
44
|
+
receiver.send :include, InstanceMethods
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
HashExtensions.included(Hash)
|
data/lib/eventhub/heartbeat.rb
CHANGED
@@ -1,77 +1,77 @@
|
|
1
|
-
module EventHub
|
2
|
-
|
3
|
-
class Heartbeat
|
4
|
-
include Helper
|
5
|
-
|
6
|
-
attr_reader :processor, :statistics, :started_at
|
7
|
-
|
8
|
-
def initialize(processor)
|
9
|
-
@started_at = Time.now
|
10
|
-
@processor = processor
|
11
|
-
@statistics = @processor.statistics
|
12
|
-
end
|
13
|
-
|
14
|
-
|
15
|
-
def build_message(action = "running")
|
16
|
-
message = ::EventHub::Message.new
|
17
|
-
message.origin_module_id = processor.name
|
18
|
-
message.origin_type = "processor"
|
19
|
-
message.origin_site_id = 'global'
|
20
|
-
|
21
|
-
message.process_name = 'event_hub.heartbeat'
|
22
|
-
|
23
|
-
now = Time.now
|
24
|
-
|
25
|
-
# message structure needs more changes
|
26
|
-
message.body = {
|
27
|
-
version: processor.version,
|
28
|
-
action: action,
|
29
|
-
pid: Process.pid,
|
30
|
-
process_name: 'event_hub.heartbeat',
|
31
|
-
|
32
|
-
heartbeat: {
|
33
|
-
started: now_stamp(started_at),
|
34
|
-
stamp_last_beat: now_stamp(now),
|
35
|
-
uptime_in_ms: (now - started_at)*1000,
|
36
|
-
heartbeat_cycle_in_ms: processor.heartbeat_cycle_in_s * 1000,
|
37
|
-
queues_consuming_from: processor.listener_queues,
|
38
|
-
queues_publishing_to: ['event_hub.inbound'], # needs more dynamic in the future
|
39
|
-
host: Socket.gethostname,
|
40
|
-
addresses: addresses,
|
41
|
-
messages: {
|
42
|
-
total: statistics.messages_total,
|
43
|
-
successful: statistics.messages_successful,
|
44
|
-
unsuccessful: statistics.messages_unsuccessful,
|
45
|
-
average_size: statistics.messages_average_size,
|
46
|
-
average_process_time_in_ms: statistics.messages_average_process_time*1000,
|
47
|
-
total_process_time_in_ms: statistics.messages_total_process_time*1000
|
48
|
-
}
|
49
|
-
}
|
50
|
-
}
|
51
|
-
message
|
52
|
-
end
|
53
|
-
|
54
|
-
private
|
55
|
-
|
56
|
-
def addresses
|
57
|
-
interfaces = Socket.getifaddrs.select do |interface|
|
58
|
-
!interface.addr.ipv4_loopback? && !interface.addr.ipv6_loopback?
|
59
|
-
end
|
60
|
-
|
61
|
-
interfaces.map do |interface|
|
62
|
-
begin
|
63
|
-
{
|
64
|
-
:interface => interface.name,
|
65
|
-
:host_name => Socket.gethostname,
|
66
|
-
:ip_address => interface.addr.ip_address
|
67
|
-
}
|
68
|
-
rescue
|
69
|
-
nil # will be ignored
|
70
|
-
end
|
71
|
-
end.compact
|
72
|
-
|
73
|
-
end
|
74
|
-
|
75
|
-
end
|
76
|
-
|
77
|
-
end
|
1
|
+
module EventHub
|
2
|
+
|
3
|
+
class Heartbeat
|
4
|
+
include Helper
|
5
|
+
|
6
|
+
attr_reader :processor, :statistics, :started_at
|
7
|
+
|
8
|
+
def initialize(processor)
|
9
|
+
@started_at = Time.now
|
10
|
+
@processor = processor
|
11
|
+
@statistics = @processor.statistics
|
12
|
+
end
|
13
|
+
|
14
|
+
|
15
|
+
def build_message(action = "running")
|
16
|
+
message = ::EventHub::Message.new
|
17
|
+
message.origin_module_id = processor.name
|
18
|
+
message.origin_type = "processor"
|
19
|
+
message.origin_site_id = 'global'
|
20
|
+
|
21
|
+
message.process_name = 'event_hub.heartbeat'
|
22
|
+
|
23
|
+
now = Time.now
|
24
|
+
|
25
|
+
# message structure needs more changes
|
26
|
+
message.body = {
|
27
|
+
version: processor.version,
|
28
|
+
action: action,
|
29
|
+
pid: Process.pid,
|
30
|
+
process_name: 'event_hub.heartbeat',
|
31
|
+
|
32
|
+
heartbeat: {
|
33
|
+
started: now_stamp(started_at),
|
34
|
+
stamp_last_beat: now_stamp(now),
|
35
|
+
uptime_in_ms: (now - started_at)*1000,
|
36
|
+
heartbeat_cycle_in_ms: processor.heartbeat_cycle_in_s * 1000,
|
37
|
+
queues_consuming_from: processor.listener_queues,
|
38
|
+
queues_publishing_to: ['event_hub.inbound'], # needs more dynamic in the future
|
39
|
+
host: Socket.gethostname,
|
40
|
+
addresses: addresses,
|
41
|
+
messages: {
|
42
|
+
total: statistics.messages_total,
|
43
|
+
successful: statistics.messages_successful,
|
44
|
+
unsuccessful: statistics.messages_unsuccessful,
|
45
|
+
average_size: statistics.messages_average_size,
|
46
|
+
average_process_time_in_ms: statistics.messages_average_process_time*1000,
|
47
|
+
total_process_time_in_ms: statistics.messages_total_process_time*1000
|
48
|
+
}
|
49
|
+
}
|
50
|
+
}
|
51
|
+
message
|
52
|
+
end
|
53
|
+
|
54
|
+
private
|
55
|
+
|
56
|
+
def addresses
|
57
|
+
interfaces = Socket.getifaddrs.select do |interface|
|
58
|
+
!interface.addr.ipv4_loopback? && !interface.addr.ipv6_loopback?
|
59
|
+
end
|
60
|
+
|
61
|
+
interfaces.map do |interface|
|
62
|
+
begin
|
63
|
+
{
|
64
|
+
:interface => interface.name,
|
65
|
+
:host_name => Socket.gethostname,
|
66
|
+
:ip_address => interface.addr.ip_address
|
67
|
+
}
|
68
|
+
rescue
|
69
|
+
nil # will be ignored
|
70
|
+
end
|
71
|
+
end.compact
|
72
|
+
|
73
|
+
end
|
74
|
+
|
75
|
+
end
|
76
|
+
|
77
|
+
end
|