eventhub-processor 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/eventhub-processor.rb +37 -37
- data/lib/eventhub/argument_parser.rb +34 -34
- data/lib/eventhub/base_exception.rb +1 -1
- data/lib/eventhub/configuration.rb +25 -25
- data/lib/eventhub/constant.rb +15 -15
- data/lib/eventhub/hash_extensions.rb +48 -48
- data/lib/eventhub/heartbeat.rb +77 -77
- data/lib/eventhub/helper.rb +49 -49
- data/lib/eventhub/message.rb +138 -138
- data/lib/eventhub/message_processor.rb +31 -31
- data/lib/eventhub/no_deadletter_exception.rb +1 -1
- data/lib/eventhub/processor.rb +269 -268
- data/lib/eventhub/statistics.rb +47 -47
- data/lib/eventhub/version.rb +3 -3
- metadata +17 -5
- data/lib/eventhub/multi_logger.rb +0 -89
- data/lib/eventhub/pidfile.rb +0 -21
- data/lib/eventhub/test.rb +0 -10
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7aa5e7a8e2f31b9683c5ff655c76c1c46bc75372
|
4
|
+
data.tar.gz: 717a76158ee4c97ba9b5595a8d21814bd36d0102
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ba810bf60a874be8e0611d2156e17a90e67a12dbfe68258d17b11ddcfdeb490fd237b8426322328d46b6bea5d818f5078c29bb5b944e3534d1f1b964dea72104
|
7
|
+
data.tar.gz: 335f306fd2c7300967fd702dace1239eca7a07b4030ee1b0be50b3f723c627704c15027e944d7c159bddac96b0e4cf440cf9574b8637f901c4f1a8a11fc81cb9
|
data/lib/eventhub-processor.rb
CHANGED
@@ -1,37 +1,37 @@
|
|
1
|
-
require 'amqp'
|
2
|
-
require 'rest-client'
|
3
|
-
require 'json'
|
4
|
-
require 'singleton'
|
5
|
-
require 'uuidtools'
|
6
|
-
require 'base64'
|
7
|
-
require 'socket'
|
8
|
-
require 'ostruct'
|
9
|
-
require 'optparse'
|
10
|
-
|
11
|
-
require_relative 'eventhub/argument_parser'
|
12
|
-
require_relative 'eventhub/version'
|
13
|
-
require_relative 'eventhub/constant'
|
14
|
-
require_relative 'eventhub/helper'
|
15
|
-
require_relative 'eventhub/multi_logger'
|
16
|
-
|
17
|
-
require_relative 'eventhub/configuration'
|
18
|
-
require_relative 'eventhub/hash_extensions'
|
19
|
-
require_relative 'eventhub/statistics'
|
20
|
-
require_relative 'eventhub/heartbeat'
|
21
|
-
require_relative 'eventhub/pidfile'
|
22
|
-
require_relative 'eventhub/processor'
|
23
|
-
require_relative 'eventhub/message_processor'
|
24
|
-
require_relative 'eventhub/message'
|
25
|
-
|
26
|
-
require_relative 'eventhub/base_exception'
|
27
|
-
require_relative 'eventhub/no_deadletter_exception'
|
28
|
-
|
29
|
-
module EventHub
|
30
|
-
def self.logger
|
31
|
-
unless @logger
|
32
|
-
@logger = MultiLogger.new
|
33
|
-
@logger.add_device(Logger.new(STDOUT))
|
34
|
-
end
|
35
|
-
@logger
|
36
|
-
end
|
37
|
-
end
|
1
|
+
require 'amqp'
|
2
|
+
require 'rest-client'
|
3
|
+
require 'json'
|
4
|
+
require 'singleton'
|
5
|
+
require 'uuidtools'
|
6
|
+
require 'base64'
|
7
|
+
require 'socket'
|
8
|
+
require 'ostruct'
|
9
|
+
require 'optparse'
|
10
|
+
|
11
|
+
require_relative 'eventhub/argument_parser'
|
12
|
+
require_relative 'eventhub/version'
|
13
|
+
require_relative 'eventhub/constant'
|
14
|
+
require_relative 'eventhub/helper'
|
15
|
+
require_relative 'eventhub/multi_logger'
|
16
|
+
|
17
|
+
require_relative 'eventhub/configuration'
|
18
|
+
require_relative 'eventhub/hash_extensions'
|
19
|
+
require_relative 'eventhub/statistics'
|
20
|
+
require_relative 'eventhub/heartbeat'
|
21
|
+
require_relative 'eventhub/pidfile'
|
22
|
+
require_relative 'eventhub/processor'
|
23
|
+
require_relative 'eventhub/message_processor'
|
24
|
+
require_relative 'eventhub/message'
|
25
|
+
|
26
|
+
require_relative 'eventhub/base_exception'
|
27
|
+
require_relative 'eventhub/no_deadletter_exception'
|
28
|
+
|
29
|
+
module EventHub
|
30
|
+
def self.logger
|
31
|
+
unless @logger
|
32
|
+
@logger = ::EventHub::Components::MultiLogger.new
|
33
|
+
@logger.add_device(Logger.new(STDOUT))
|
34
|
+
end
|
35
|
+
@logger
|
36
|
+
end
|
37
|
+
end
|
@@ -1,35 +1,35 @@
|
|
1
|
-
module EventHub
|
2
|
-
|
3
|
-
class ArgumentParser
|
4
|
-
|
5
|
-
def self.parse(args)
|
6
|
-
# The options specified on the command line will be collected in *options*.
|
7
|
-
# We set default values here.
|
8
|
-
options = OpenStruct.new
|
9
|
-
options.environment = 'development'
|
10
|
-
options.detached = false
|
11
|
-
|
12
|
-
opt_parser = OptionParser.new do |opts|
|
13
|
-
opts.banner = "Usage: #{args[0]}.rb [options]"
|
14
|
-
yield(opts, options) if block_given? # allow to add more options
|
15
|
-
|
16
|
-
opts.on("-e", "--environment ENVIRONMENT","Environment the processor is running") do |environment|
|
17
|
-
options.environment = environment
|
18
|
-
end
|
19
|
-
|
20
|
-
opts.on("-d", "--detached", "Run processor detached as a daemon") do |v|
|
21
|
-
options.detached = v
|
22
|
-
end
|
23
|
-
|
24
|
-
opts.on_tail("-h", "--help", "Show this message") do
|
25
|
-
puts opts
|
26
|
-
exit
|
27
|
-
end
|
28
|
-
end
|
29
|
-
|
30
|
-
opt_parser.parse!(args)
|
31
|
-
options
|
32
|
-
end
|
33
|
-
end
|
34
|
-
|
1
|
+
module EventHub
|
2
|
+
|
3
|
+
class ArgumentParser
|
4
|
+
|
5
|
+
def self.parse(args)
|
6
|
+
# The options specified on the command line will be collected in *options*.
|
7
|
+
# We set default values here.
|
8
|
+
options = OpenStruct.new
|
9
|
+
options.environment = 'development'
|
10
|
+
options.detached = false
|
11
|
+
|
12
|
+
opt_parser = OptionParser.new do |opts|
|
13
|
+
opts.banner = "Usage: #{args[0]}.rb [options]"
|
14
|
+
yield(opts, options) if block_given? # allow to add more options
|
15
|
+
|
16
|
+
opts.on("-e", "--environment ENVIRONMENT","Environment the processor is running") do |environment|
|
17
|
+
options.environment = environment
|
18
|
+
end
|
19
|
+
|
20
|
+
opts.on("-d", "--detached", "Run processor detached as a daemon") do |v|
|
21
|
+
options.detached = v
|
22
|
+
end
|
23
|
+
|
24
|
+
opts.on_tail("-h", "--help", "Show this message") do
|
25
|
+
puts opts
|
26
|
+
exit
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
opt_parser.parse!(args)
|
31
|
+
options
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
35
|
end
|
@@ -1,2 +1,2 @@
|
|
1
|
-
class EventHub::BaseException < RuntimeError
|
1
|
+
class EventHub::BaseException < RuntimeError
|
2
2
|
end
|
@@ -1,26 +1,26 @@
|
|
1
|
-
module EventHub
|
2
|
-
|
3
|
-
class Configuration
|
4
|
-
include Singleton
|
5
|
-
include Helper
|
6
|
-
|
7
|
-
attr_accessor :data, :folder, :environment
|
8
|
-
|
9
|
-
def initialize
|
10
|
-
@data = nil
|
11
|
-
@environment = 'development'
|
12
|
-
end
|
13
|
-
|
14
|
-
def load_file(input, env = 'development')
|
15
|
-
json = JSON.parse(IO.read(input))
|
16
|
-
@data = json[env]
|
17
|
-
@environment = env
|
18
|
-
true
|
19
|
-
rescue => e
|
20
|
-
EventHub.logger.info("Unexpected exception while loading configuration [#{input}]: #{format_string(e.message)}")
|
21
|
-
false
|
22
|
-
end
|
23
|
-
|
24
|
-
end
|
25
|
-
|
1
|
+
module EventHub
|
2
|
+
|
3
|
+
class Configuration
|
4
|
+
include Singleton
|
5
|
+
include Helper
|
6
|
+
|
7
|
+
attr_accessor :data, :folder, :environment
|
8
|
+
|
9
|
+
def initialize
|
10
|
+
@data = nil
|
11
|
+
@environment = 'development'
|
12
|
+
end
|
13
|
+
|
14
|
+
def load_file(input, env = 'development')
|
15
|
+
json = JSON.parse(IO.read(input))
|
16
|
+
@data = json[env]
|
17
|
+
@environment = env
|
18
|
+
true
|
19
|
+
rescue => e
|
20
|
+
EventHub.logger.info("Unexpected exception while loading configuration [#{input}]: #{format_string(e.message)}")
|
21
|
+
false
|
22
|
+
end
|
23
|
+
|
24
|
+
end
|
25
|
+
|
26
26
|
end
|
data/lib/eventhub/constant.rb
CHANGED
@@ -1,16 +1,16 @@
|
|
1
|
-
module EventHub
|
2
|
-
|
3
|
-
EH_X_INBOUND = 'event_hub.inbound'
|
4
|
-
|
5
|
-
STATUS_INITIAL = 0 # To be set when dispatcher needs to dispatch to first process step.
|
6
|
-
STATUS_SUCCESS = 200 # To be set to indicate successful processed message. Dispatcher will routes message to the next step.
|
7
|
-
STATUS_RETRY = 300 # To be set to trigger retry cycle controlled by the dispatcher
|
8
|
-
STATUS_RETRY_PENDING = 301 # Set and used by the dispatcher only.
|
9
|
-
# Set before putting the message into a retry queue.
|
10
|
-
# Once message has been retried it will sent do the same step with status.code = STATUS_SUCCESS
|
11
|
-
STATUS_INVALID = 400 # To be set to indicate invalid message (not json, invalid Event Hub Message).
|
12
|
-
# Dispatcher will publish message to the invalid queue.
|
13
|
-
STATUS_DEADLETTER = 500 # To be set by dispatcher, processor or channel adapters to indicate
|
14
|
-
# that message needs to be dead-lettered. Rejected messages could miss the
|
15
|
-
# status.code = STATUS_DEADLETTER due to the RabbitMQ deadletter exchange mechanism.
|
1
|
+
module EventHub
|
2
|
+
|
3
|
+
EH_X_INBOUND = 'event_hub.inbound'
|
4
|
+
|
5
|
+
STATUS_INITIAL = 0 # To be set when dispatcher needs to dispatch to first process step.
|
6
|
+
STATUS_SUCCESS = 200 # To be set to indicate successful processed message. Dispatcher will routes message to the next step.
|
7
|
+
STATUS_RETRY = 300 # To be set to trigger retry cycle controlled by the dispatcher
|
8
|
+
STATUS_RETRY_PENDING = 301 # Set and used by the dispatcher only.
|
9
|
+
# Set before putting the message into a retry queue.
|
10
|
+
# Once message has been retried it will sent do the same step with status.code = STATUS_SUCCESS
|
11
|
+
STATUS_INVALID = 400 # To be set to indicate invalid message (not json, invalid Event Hub Message).
|
12
|
+
# Dispatcher will publish message to the invalid queue.
|
13
|
+
STATUS_DEADLETTER = 500 # To be set by dispatcher, processor or channel adapters to indicate
|
14
|
+
# that message needs to be dead-lettered. Rejected messages could miss the
|
15
|
+
# status.code = STATUS_DEADLETTER due to the RabbitMQ deadletter exchange mechanism.
|
16
16
|
end
|
@@ -1,48 +1,48 @@
|
|
1
|
-
module HashExtensions
|
2
|
-
|
3
|
-
module ClassMethods
|
4
|
-
end
|
5
|
-
|
6
|
-
module InstanceMethods
|
7
|
-
# get value from provided key path, e.g. hash.get(%w(event_hub plate.queue1 retry_s))
|
8
|
-
# "a" => { "b" => { "c" => { "value"}}}
|
9
|
-
def get(arg)
|
10
|
-
path = arg.is_a?(String) ? arg.split('.') : arg
|
11
|
-
path.inject(self,:[])
|
12
|
-
rescue NoMethodError
|
13
|
-
return nil
|
14
|
-
end
|
15
|
-
|
16
|
-
# set value from provided key path, e.h. hash.set('a.b.c','new value')
|
17
|
-
# if overwrite is false, value will be set if it was nil previously
|
18
|
-
def set(arg,value,overwrite=true)
|
19
|
-
*key_path, last = arg.is_a?(String) ? arg.split(".") : arg
|
20
|
-
if overwrite
|
21
|
-
key_path.inject(self) { |h,key| h.has_key?(key) ? h[key] : h[key]={}} [last] = value
|
22
|
-
else
|
23
|
-
key_path.inject(self) { |h,key| h.has_key?(key) ? h[key] : h[key]={}} [last] ||= value
|
24
|
-
end
|
25
|
-
end
|
26
|
-
|
27
|
-
# get all keys path, { 'a' => 'value1', 'b' => { 'c' => 'value2'}}.all_keys_with_path => ['a','b.c']
|
28
|
-
def all_keys_with_path(parent=nil)
|
29
|
-
a = []
|
30
|
-
each do |k,v|
|
31
|
-
if v.is_a?(Hash)
|
32
|
-
a << v.all_keys_with_path([parent,k].compact.join('.'))
|
33
|
-
else
|
34
|
-
a << "#{[parent,k].compact.join(".")}"
|
35
|
-
end
|
36
|
-
end
|
37
|
-
a.flatten
|
38
|
-
end
|
39
|
-
|
40
|
-
end
|
41
|
-
|
42
|
-
def self.included(receiver)
|
43
|
-
receiver.extend ClassMethods
|
44
|
-
receiver.send :include, InstanceMethods
|
45
|
-
end
|
46
|
-
end
|
47
|
-
|
48
|
-
HashExtensions.included(Hash)
|
1
|
+
module HashExtensions
|
2
|
+
|
3
|
+
module ClassMethods
|
4
|
+
end
|
5
|
+
|
6
|
+
module InstanceMethods
|
7
|
+
# get value from provided key path, e.g. hash.get(%w(event_hub plate.queue1 retry_s))
|
8
|
+
# "a" => { "b" => { "c" => { "value"}}}
|
9
|
+
def get(arg)
|
10
|
+
path = arg.is_a?(String) ? arg.split('.') : arg
|
11
|
+
path.inject(self,:[])
|
12
|
+
rescue NoMethodError
|
13
|
+
return nil
|
14
|
+
end
|
15
|
+
|
16
|
+
# set value from provided key path, e.h. hash.set('a.b.c','new value')
|
17
|
+
# if overwrite is false, value will be set if it was nil previously
|
18
|
+
def set(arg,value,overwrite=true)
|
19
|
+
*key_path, last = arg.is_a?(String) ? arg.split(".") : arg
|
20
|
+
if overwrite
|
21
|
+
key_path.inject(self) { |h,key| h.has_key?(key) ? h[key] : h[key]={}} [last] = value
|
22
|
+
else
|
23
|
+
key_path.inject(self) { |h,key| h.has_key?(key) ? h[key] : h[key]={}} [last] ||= value
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
# get all keys path, { 'a' => 'value1', 'b' => { 'c' => 'value2'}}.all_keys_with_path => ['a','b.c']
|
28
|
+
def all_keys_with_path(parent=nil)
|
29
|
+
a = []
|
30
|
+
each do |k,v|
|
31
|
+
if v.is_a?(Hash)
|
32
|
+
a << v.all_keys_with_path([parent,k].compact.join('.'))
|
33
|
+
else
|
34
|
+
a << "#{[parent,k].compact.join(".")}"
|
35
|
+
end
|
36
|
+
end
|
37
|
+
a.flatten
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
41
|
+
|
42
|
+
def self.included(receiver)
|
43
|
+
receiver.extend ClassMethods
|
44
|
+
receiver.send :include, InstanceMethods
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
HashExtensions.included(Hash)
|
data/lib/eventhub/heartbeat.rb
CHANGED
@@ -1,77 +1,77 @@
|
|
1
|
-
module EventHub
|
2
|
-
|
3
|
-
class Heartbeat
|
4
|
-
include Helper
|
5
|
-
|
6
|
-
attr_reader :processor, :statistics, :started_at
|
7
|
-
|
8
|
-
def initialize(processor)
|
9
|
-
@started_at = Time.now
|
10
|
-
@processor = processor
|
11
|
-
@statistics = @processor.statistics
|
12
|
-
end
|
13
|
-
|
14
|
-
|
15
|
-
def build_message(action = "running")
|
16
|
-
message = ::EventHub::Message.new
|
17
|
-
message.origin_module_id = processor.name
|
18
|
-
message.origin_type = "processor"
|
19
|
-
message.origin_site_id = 'global'
|
20
|
-
|
21
|
-
message.process_name = 'event_hub.heartbeat'
|
22
|
-
|
23
|
-
now = Time.now
|
24
|
-
|
25
|
-
# message structure needs more changes
|
26
|
-
message.body = {
|
27
|
-
version: processor.version,
|
28
|
-
action: action,
|
29
|
-
pid: Process.pid,
|
30
|
-
process_name: 'event_hub.heartbeat',
|
31
|
-
|
32
|
-
heartbeat: {
|
33
|
-
started: now_stamp(started_at),
|
34
|
-
stamp_last_beat: now_stamp(now),
|
35
|
-
uptime_in_ms: (now - started_at)*1000,
|
36
|
-
heartbeat_cycle_in_ms: processor.heartbeat_cycle_in_s * 1000,
|
37
|
-
queues_consuming_from: processor.listener_queues,
|
38
|
-
queues_publishing_to: ['event_hub.inbound'], # needs more dynamic in the future
|
39
|
-
host: Socket.gethostname,
|
40
|
-
addresses: addresses,
|
41
|
-
messages: {
|
42
|
-
total: statistics.messages_total,
|
43
|
-
successful: statistics.messages_successful,
|
44
|
-
unsuccessful: statistics.messages_unsuccessful,
|
45
|
-
average_size: statistics.messages_average_size,
|
46
|
-
average_process_time_in_ms: statistics.messages_average_process_time*1000,
|
47
|
-
total_process_time_in_ms: statistics.messages_total_process_time*1000
|
48
|
-
}
|
49
|
-
}
|
50
|
-
}
|
51
|
-
message
|
52
|
-
end
|
53
|
-
|
54
|
-
private
|
55
|
-
|
56
|
-
def addresses
|
57
|
-
interfaces = Socket.getifaddrs.select do |interface|
|
58
|
-
!interface.addr.ipv4_loopback? && !interface.addr.ipv6_loopback?
|
59
|
-
end
|
60
|
-
|
61
|
-
interfaces.map do |interface|
|
62
|
-
begin
|
63
|
-
{
|
64
|
-
:interface => interface.name,
|
65
|
-
:host_name => Socket.gethostname,
|
66
|
-
:ip_address => interface.addr.ip_address
|
67
|
-
}
|
68
|
-
rescue
|
69
|
-
nil # will be ignored
|
70
|
-
end
|
71
|
-
end.compact
|
72
|
-
|
73
|
-
end
|
74
|
-
|
75
|
-
end
|
76
|
-
|
77
|
-
end
|
1
|
+
module EventHub
|
2
|
+
|
3
|
+
class Heartbeat
|
4
|
+
include Helper
|
5
|
+
|
6
|
+
attr_reader :processor, :statistics, :started_at
|
7
|
+
|
8
|
+
def initialize(processor)
|
9
|
+
@started_at = Time.now
|
10
|
+
@processor = processor
|
11
|
+
@statistics = @processor.statistics
|
12
|
+
end
|
13
|
+
|
14
|
+
|
15
|
+
def build_message(action = "running")
|
16
|
+
message = ::EventHub::Message.new
|
17
|
+
message.origin_module_id = processor.name
|
18
|
+
message.origin_type = "processor"
|
19
|
+
message.origin_site_id = 'global'
|
20
|
+
|
21
|
+
message.process_name = 'event_hub.heartbeat'
|
22
|
+
|
23
|
+
now = Time.now
|
24
|
+
|
25
|
+
# message structure needs more changes
|
26
|
+
message.body = {
|
27
|
+
version: processor.version,
|
28
|
+
action: action,
|
29
|
+
pid: Process.pid,
|
30
|
+
process_name: 'event_hub.heartbeat',
|
31
|
+
|
32
|
+
heartbeat: {
|
33
|
+
started: now_stamp(started_at),
|
34
|
+
stamp_last_beat: now_stamp(now),
|
35
|
+
uptime_in_ms: (now - started_at)*1000,
|
36
|
+
heartbeat_cycle_in_ms: processor.heartbeat_cycle_in_s * 1000,
|
37
|
+
queues_consuming_from: processor.listener_queues,
|
38
|
+
queues_publishing_to: ['event_hub.inbound'], # needs more dynamic in the future
|
39
|
+
host: Socket.gethostname,
|
40
|
+
addresses: addresses,
|
41
|
+
messages: {
|
42
|
+
total: statistics.messages_total,
|
43
|
+
successful: statistics.messages_successful,
|
44
|
+
unsuccessful: statistics.messages_unsuccessful,
|
45
|
+
average_size: statistics.messages_average_size,
|
46
|
+
average_process_time_in_ms: statistics.messages_average_process_time*1000,
|
47
|
+
total_process_time_in_ms: statistics.messages_total_process_time*1000
|
48
|
+
}
|
49
|
+
}
|
50
|
+
}
|
51
|
+
message
|
52
|
+
end
|
53
|
+
|
54
|
+
private
|
55
|
+
|
56
|
+
def addresses
|
57
|
+
interfaces = Socket.getifaddrs.select do |interface|
|
58
|
+
!interface.addr.ipv4_loopback? && !interface.addr.ipv6_loopback?
|
59
|
+
end
|
60
|
+
|
61
|
+
interfaces.map do |interface|
|
62
|
+
begin
|
63
|
+
{
|
64
|
+
:interface => interface.name,
|
65
|
+
:host_name => Socket.gethostname,
|
66
|
+
:ip_address => interface.addr.ip_address
|
67
|
+
}
|
68
|
+
rescue
|
69
|
+
nil # will be ignored
|
70
|
+
end
|
71
|
+
end.compact
|
72
|
+
|
73
|
+
end
|
74
|
+
|
75
|
+
end
|
76
|
+
|
77
|
+
end
|