eventhub-processor2 1.8.0 → 1.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rspec +1 -0
- data/.ruby-version +1 -1
- data/.travis.yml +2 -2
- data/CHANGELOG.md +6 -0
- data/Gemfile +1 -1
- data/Gemfile.lock +57 -41
- data/Rakefile +7 -4
- data/docker/Dockerfile +1 -1
- data/docker/docker-compose.yml +1 -1
- data/eventhub-processor2.gemspec +24 -23
- data/example/crasher.rb +12 -13
- data/example/publisher.rb +41 -41
- data/example/receiver.rb +2 -2
- data/example/router.rb +3 -3
- data/lib/eventhub/actor_heartbeat.rb +23 -23
- data/lib/eventhub/actor_listener.rb +17 -21
- data/lib/eventhub/actor_publisher.rb +7 -7
- data/lib/eventhub/actor_watchdog.rb +3 -4
- data/lib/eventhub/base.rb +24 -31
- data/lib/eventhub/base_exception.rb +2 -2
- data/lib/eventhub/configuration.rb +30 -25
- data/lib/eventhub/constant.rb +24 -24
- data/lib/eventhub/hash_extensions.rb +8 -8
- data/lib/eventhub/helper.rb +7 -7
- data/lib/eventhub/logger.rb +2 -2
- data/lib/eventhub/message.rb +45 -45
- data/lib/eventhub/processor2.rb +22 -22
- data/lib/eventhub/sleeper.rb +1 -1
- data/lib/eventhub/statistics.rb +6 -8
- data/lib/eventhub/version.rb +1 -1
- metadata +24 -10
data/example/receiver.rb
CHANGED
@@ -1,9 +1,9 @@
|
|
1
|
-
require_relative
|
1
|
+
require_relative "../lib/eventhub/base"
|
2
2
|
|
3
3
|
module EventHub
|
4
4
|
class Receiver < Processor2
|
5
5
|
def handle_message(message, args = {})
|
6
|
-
id = message.body[
|
6
|
+
id = message.body["id"]
|
7
7
|
EventHub.logger.info("[#{id}] - Received")
|
8
8
|
|
9
9
|
file_name = "data/#{id}.json"
|
data/example/router.rb
CHANGED
@@ -1,12 +1,12 @@
|
|
1
|
-
require_relative
|
1
|
+
require_relative "../lib/eventhub/base"
|
2
2
|
|
3
3
|
module EventHub
|
4
4
|
# Demo class
|
5
5
|
class Router < Processor2
|
6
6
|
def handle_message(message, args = {})
|
7
|
-
id = message.body[
|
7
|
+
id = message.body["id"]
|
8
8
|
EventHub.logger.info("Received: [#{id}]")
|
9
|
-
publish(message: message.to_json, exchange_name:
|
9
|
+
publish(message: message.to_json, exchange_name: "example.inbound")
|
10
10
|
EventHub.logger.info("Returned: [#{id}]")
|
11
11
|
nil
|
12
12
|
end
|
@@ -12,23 +12,23 @@ module EventHub
|
|
12
12
|
end
|
13
13
|
|
14
14
|
def start
|
15
|
-
EventHub.logger.info(
|
15
|
+
EventHub.logger.info("Heartbeat is starting...")
|
16
16
|
|
17
|
-
every(
|
17
|
+
every(60 * 60 * 24) { EventHub.logger.info("Actual actors: #{Celluloid::Actor.all.size}: #{Celluloid::Actor.all.map { |a| a.class }.join(", ")}") }
|
18
18
|
|
19
|
-
publish(heartbeat(action:
|
20
|
-
EventHub.logger.info(
|
19
|
+
publish(heartbeat(action: "started"))
|
20
|
+
EventHub.logger.info("Heartbeat has sent [started] beat")
|
21
21
|
loop do
|
22
22
|
sleep Configuration.processor[:heartbeat_cycle_in_s]
|
23
23
|
publish(heartbeat)
|
24
|
-
EventHub.logger.info(
|
24
|
+
EventHub.logger.info("Heartbeat has sent a beat")
|
25
25
|
end
|
26
26
|
end
|
27
27
|
|
28
28
|
def cleanup
|
29
|
-
EventHub.logger.info(
|
30
|
-
publish(heartbeat(action:
|
31
|
-
EventHub.logger.info(
|
29
|
+
EventHub.logger.info("Heartbeat is cleaning up...")
|
30
|
+
publish(heartbeat(action: "stopped"))
|
31
|
+
EventHub.logger.info("Heartbeat has sent a [stopped] beat")
|
32
32
|
end
|
33
33
|
|
34
34
|
private
|
@@ -43,29 +43,29 @@ module EventHub
|
|
43
43
|
success = channel.wait_for_confirms
|
44
44
|
|
45
45
|
unless success
|
46
|
-
raise
|
47
|
-
|
46
|
+
raise "Published heartbeat message has "\
|
47
|
+
"not been confirmed by the server"
|
48
48
|
end
|
49
49
|
ensure
|
50
|
-
connection
|
50
|
+
connection&.close
|
51
51
|
end
|
52
52
|
|
53
|
-
def heartbeat(args = {
|
53
|
+
def heartbeat(args = {action: "running"})
|
54
54
|
message = EventHub::Message.new
|
55
|
-
message.origin_module_id
|
56
|
-
message.origin_type
|
57
|
-
message.origin_site_id
|
55
|
+
message.origin_module_id = EventHub::Configuration.name
|
56
|
+
message.origin_type = "processor"
|
57
|
+
message.origin_site_id = "global"
|
58
58
|
|
59
|
-
message.process_name
|
59
|
+
message.process_name = "event_hub.heartbeat"
|
60
60
|
|
61
61
|
now = Time.now
|
62
62
|
|
63
63
|
# message structure needs more changes
|
64
64
|
message.body = {
|
65
65
|
version: @processor_instance.send(:version),
|
66
|
-
action:
|
67
|
-
pid:
|
68
|
-
process_name:
|
66
|
+
action: args[:action],
|
67
|
+
pid: Process.pid,
|
68
|
+
process_name: "event_hub.heartbeat",
|
69
69
|
heartbeat: {
|
70
70
|
started: now_stamp(started_at),
|
71
71
|
stamp_last_beat: now_stamp(now),
|
@@ -90,11 +90,11 @@ module EventHub
|
|
90
90
|
end
|
91
91
|
|
92
92
|
def addresses
|
93
|
-
interfaces = Socket.getifaddrs.select
|
93
|
+
interfaces = Socket.getifaddrs.select { |interface|
|
94
94
|
!interface.addr.ipv4_loopback? && !interface.addr.ipv6_loopback?
|
95
|
-
|
95
|
+
}
|
96
96
|
|
97
|
-
interfaces.map
|
97
|
+
interfaces.map { |interface|
|
98
98
|
begin
|
99
99
|
{
|
100
100
|
interface: interface.name,
|
@@ -104,7 +104,7 @@ module EventHub
|
|
104
104
|
rescue
|
105
105
|
nil # will be ignored
|
106
106
|
end
|
107
|
-
|
107
|
+
}.compact
|
108
108
|
end
|
109
109
|
|
110
110
|
def messages_statistics
|
@@ -15,14 +15,14 @@ module EventHub
|
|
15
15
|
end
|
16
16
|
|
17
17
|
def start
|
18
|
-
EventHub.logger.info(
|
18
|
+
EventHub.logger.info("Listener is starting...")
|
19
19
|
EventHub::Configuration.processor[:listener_queues].each_with_index do |queue_name, index|
|
20
20
|
async.listen(queue_name: queue_name, index: index)
|
21
21
|
end
|
22
22
|
end
|
23
23
|
|
24
24
|
def restart
|
25
|
-
raise
|
25
|
+
raise "Listener is restarting..."
|
26
26
|
end
|
27
27
|
|
28
28
|
def listen(args = {})
|
@@ -30,7 +30,7 @@ module EventHub
|
|
30
30
|
EventHub.logger.info("Listening to queue [#{queue_name}]")
|
31
31
|
consumer.on_delivery do |delivery_info, metadata, payload|
|
32
32
|
EventHub.logger.info("#{queue_name}: [#{delivery_info.delivery_tag}]"\
|
33
|
-
|
33
|
+
" delivery")
|
34
34
|
|
35
35
|
@processor_instance.statistics.measure(payload.size) do
|
36
36
|
handle_payload(payload: payload,
|
@@ -38,25 +38,21 @@ module EventHub
|
|
38
38
|
queue_name: queue_name,
|
39
39
|
content_type: metadata[:content_type],
|
40
40
|
priority: metadata[:priority],
|
41
|
-
delivery_tag: delivery_info.delivery_tag
|
42
|
-
)
|
41
|
+
delivery_tag: delivery_info.delivery_tag)
|
43
42
|
channel.acknowledge(delivery_info.delivery_tag, false)
|
44
43
|
end
|
45
44
|
|
46
45
|
EventHub.logger.info("#{queue_name}: [#{delivery_info.delivery_tag}]"\
|
47
|
-
|
46
|
+
" acknowledged")
|
48
47
|
end
|
49
48
|
queue.subscribe_with(consumer, block: false)
|
50
49
|
end
|
51
|
-
|
52
50
|
rescue => error
|
53
51
|
EventHub.logger.error("Unexpected exception: #{error}. It should restart now with this exception...")
|
54
52
|
raise
|
55
53
|
end
|
56
54
|
|
57
55
|
def with_listen(args = {}, &block)
|
58
|
-
connection_string, connection_properties = connection_properties
|
59
|
-
|
60
56
|
connection = create_bunny_connection
|
61
57
|
connection.start
|
62
58
|
queue_name = args[:queue_name]
|
@@ -65,11 +61,11 @@ module EventHub
|
|
65
61
|
channel.prefetch(1)
|
66
62
|
queue = channel.queue(queue_name, durable: true)
|
67
63
|
consumer = EventHub::Consumer.new(channel,
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
64
|
+
queue,
|
65
|
+
EventHub::Configuration.name +
|
66
|
+
"-" +
|
67
|
+
args[:index].to_s,
|
68
|
+
false)
|
73
69
|
yield connection, channel, consumer, queue, queue_name
|
74
70
|
end
|
75
71
|
|
@@ -86,18 +82,18 @@ module EventHub
|
|
86
82
|
# return invalid messages to dispatcher
|
87
83
|
if message.invalid?
|
88
84
|
response_messages << message
|
89
|
-
EventHub.logger.info("-> #{message
|
85
|
+
EventHub.logger.info("-> #{message} => return invalid to dispatcher")
|
90
86
|
else
|
91
87
|
begin
|
92
88
|
response_messages = @processor_instance.send(:handle_message,
|
93
|
-
|
94
|
-
|
89
|
+
message,
|
90
|
+
pass_arguments(args))
|
95
91
|
rescue => exception
|
96
92
|
# this catches unexpected exceptions in handle message method
|
97
93
|
# deadletter the message via dispatcher
|
98
94
|
message.status_code = EventHub::STATUS_DEADLETTER
|
99
95
|
message.status_message = exception.to_s
|
100
|
-
EventHub.logger.info("-> #{message
|
96
|
+
EventHub.logger.info("-> #{message} => return exception to dispatcher")
|
101
97
|
response_messages << message
|
102
98
|
end
|
103
99
|
end
|
@@ -109,15 +105,15 @@ module EventHub
|
|
109
105
|
|
110
106
|
def pass_arguments(args = {})
|
111
107
|
keys_to_pass = [:queue_name, :content_type, :priority, :delivery_tag]
|
112
|
-
args.select{ |key| keys_to_pass.include?(key) }
|
108
|
+
args.select { |key| keys_to_pass.include?(key) }
|
113
109
|
end
|
114
110
|
|
115
111
|
def cleanup
|
116
|
-
EventHub.logger.info(
|
112
|
+
EventHub.logger.info("Listener is cleaning up...")
|
117
113
|
# close all open connections
|
118
114
|
return unless @connections
|
119
115
|
@connections.values.each do |connection|
|
120
|
-
connection
|
116
|
+
connection&.close
|
121
117
|
end
|
122
118
|
end
|
123
119
|
|
@@ -7,7 +7,7 @@ module EventHub
|
|
7
7
|
finalizer :cleanup
|
8
8
|
|
9
9
|
def initialize
|
10
|
-
EventHub.logger.info(
|
10
|
+
EventHub.logger.info("Publisher is starting...")
|
11
11
|
@connection = nil
|
12
12
|
end
|
13
13
|
|
@@ -31,16 +31,16 @@ module EventHub
|
|
31
31
|
success = channel.wait_for_confirms
|
32
32
|
|
33
33
|
unless success
|
34
|
-
raise
|
35
|
-
|
34
|
+
raise "Published message from Listener actor "\
|
35
|
+
"has not been confirmed by the server"
|
36
36
|
end
|
37
|
-
|
38
|
-
|
37
|
+
ensure
|
38
|
+
channel&.close
|
39
39
|
end
|
40
40
|
|
41
41
|
def cleanup
|
42
|
-
EventHub.logger.info(
|
43
|
-
@connection
|
42
|
+
EventHub.logger.info("Publisher is cleaning up...")
|
43
|
+
@connection&.close
|
44
44
|
end
|
45
45
|
end
|
46
46
|
end
|
@@ -7,20 +7,19 @@ module EventHub
|
|
7
7
|
finalizer :cleanup
|
8
8
|
|
9
9
|
def initialize
|
10
|
-
EventHub.logger.info(
|
10
|
+
EventHub.logger.info("Watchdog is starting...")
|
11
11
|
async.start
|
12
12
|
end
|
13
13
|
|
14
14
|
def start
|
15
15
|
loop do
|
16
|
-
EventHub.logger.info('Running watchdog...')
|
17
16
|
watch
|
18
17
|
sleep Configuration.processor[:watchdog_cycle_in_s]
|
19
18
|
end
|
20
19
|
end
|
21
20
|
|
22
21
|
def cleanup
|
23
|
-
EventHub.logger.info(
|
22
|
+
EventHub.logger.info("Watchdog is cleaning up...")
|
24
23
|
end
|
25
24
|
|
26
25
|
private
|
@@ -36,7 +35,7 @@ module EventHub
|
|
36
35
|
end
|
37
36
|
end
|
38
37
|
ensure
|
39
|
-
connection
|
38
|
+
connection&.close
|
40
39
|
end
|
41
40
|
end
|
42
41
|
end
|
data/lib/eventhub/base.rb
CHANGED
@@ -1,36 +1,29 @@
|
|
1
|
-
require
|
2
|
-
require
|
3
|
-
require
|
4
|
-
require
|
1
|
+
require "uuidtools"
|
2
|
+
require "json"
|
3
|
+
require "base64"
|
4
|
+
require "optparse"
|
5
5
|
|
6
|
-
require
|
7
|
-
require
|
8
|
-
require
|
6
|
+
require "eventhub/components"
|
7
|
+
require "logstash-logger"
|
8
|
+
require "bunny"
|
9
|
+
require "celluloid"
|
9
10
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
require_relative
|
19
|
-
require_relative
|
20
|
-
require_relative
|
21
|
-
require_relative
|
22
|
-
require_relative
|
23
|
-
require_relative
|
24
|
-
require_relative
|
25
|
-
require_relative
|
26
|
-
require_relative 'message'
|
27
|
-
require_relative 'statistics'
|
28
|
-
require_relative 'consumer'
|
29
|
-
require_relative 'actor_heartbeat'
|
30
|
-
require_relative 'actor_watchdog'
|
31
|
-
require_relative 'actor_publisher'
|
32
|
-
require_relative 'actor_listener'
|
33
|
-
require_relative 'processor2'
|
11
|
+
require_relative "version"
|
12
|
+
require_relative "constant"
|
13
|
+
require_relative "base_exception"
|
14
|
+
require_relative "logger"
|
15
|
+
require_relative "helper"
|
16
|
+
require_relative "sleeper"
|
17
|
+
require_relative "hash_extensions"
|
18
|
+
require_relative "configuration"
|
19
|
+
require_relative "message"
|
20
|
+
require_relative "statistics"
|
21
|
+
require_relative "consumer"
|
22
|
+
require_relative "actor_heartbeat"
|
23
|
+
require_relative "actor_watchdog"
|
24
|
+
require_relative "actor_publisher"
|
25
|
+
require_relative "actor_listener"
|
26
|
+
require_relative "processor2"
|
34
27
|
|
35
28
|
Celluloid.logger = nil
|
36
29
|
Celluloid.exception_handler { |ex| EventHub.logger.error "Exception occured: #{ex}" }
|
@@ -3,9 +3,9 @@ module EventHub
|
|
3
3
|
# BaseException class
|
4
4
|
class BaseException < RuntimeError
|
5
5
|
attr_accessor :code, :message
|
6
|
-
def initialize(message=nil, code=EventHub::STATUS_DEADLETTER)
|
6
|
+
def initialize(message = nil, code = EventHub::STATUS_DEADLETTER)
|
7
7
|
@message = message
|
8
|
-
@code
|
8
|
+
@code = code
|
9
9
|
super(@message)
|
10
10
|
end
|
11
11
|
end
|
@@ -6,16 +6,16 @@ module EventHub
|
|
6
6
|
extend self
|
7
7
|
extend Helper
|
8
8
|
|
9
|
-
attr_reader :name
|
10
|
-
attr_reader :environment
|
11
|
-
attr_reader :detached
|
12
|
-
attr_reader :config_file
|
13
|
-
attr_reader :config_data
|
14
|
-
|
15
|
-
@name =
|
16
|
-
@environment =
|
9
|
+
attr_reader :name # name of processor
|
10
|
+
attr_reader :environment # environment the processor is running
|
11
|
+
attr_reader :detached # run processor run as a daemon
|
12
|
+
attr_reader :config_file # name of configuration file
|
13
|
+
attr_reader :config_data # data from configuration file
|
14
|
+
|
15
|
+
@name = "undefined"
|
16
|
+
@environment = "development"
|
17
17
|
@detached = false
|
18
|
-
@config_file = File.join(Dir.getwd,
|
18
|
+
@config_file = File.join(Dir.getwd, "config", "#{@name}.json")
|
19
19
|
@config_data = {}
|
20
20
|
|
21
21
|
# set name of processor
|
@@ -24,32 +24,32 @@ module EventHub
|
|
24
24
|
end
|
25
25
|
|
26
26
|
def reset
|
27
|
-
@name =
|
28
|
-
@environment =
|
27
|
+
@name = "undefined"
|
28
|
+
@environment = "development"
|
29
29
|
@detached = false
|
30
|
-
@config_file = File.join(Dir.getwd,
|
30
|
+
@config_file = File.join(Dir.getwd, "config", "#{@name}.json")
|
31
31
|
@config_data = {}
|
32
32
|
end
|
33
33
|
|
34
34
|
# parse options from argument list
|
35
35
|
def parse_options(argv = ARGV)
|
36
|
-
@config_file = File.join(Dir.getwd,
|
36
|
+
@config_file = File.join(Dir.getwd, "config", "#{@name}.json")
|
37
37
|
|
38
|
-
OptionParser.new
|
39
|
-
note =
|
40
|
-
opts.on(
|
38
|
+
OptionParser.new { |opts|
|
39
|
+
note = "Define environment"
|
40
|
+
opts.on("-e", "--environment ENVIRONMENT", note) do |environment|
|
41
41
|
@environment = environment
|
42
42
|
end
|
43
43
|
|
44
|
-
opts.on(
|
44
|
+
opts.on("-d", "--detached", "Run processor detached as a daemon") do
|
45
45
|
@detached = true
|
46
46
|
end
|
47
47
|
|
48
|
-
note =
|
49
|
-
opts.on(
|
48
|
+
note = "Define configuration file"
|
49
|
+
opts.on("-c", "--config CONFIG", note) do |config|
|
50
50
|
@config_file = config
|
51
51
|
end
|
52
|
-
|
52
|
+
}.parse!(argv)
|
53
53
|
|
54
54
|
true
|
55
55
|
rescue OptionParser::InvalidOption => e
|
@@ -71,7 +71,7 @@ module EventHub
|
|
71
71
|
new_data = JSON.parse(File.read(@config_file), symbolize_names: true)
|
72
72
|
rescue => e
|
73
73
|
EventHub.logger.warn("Exception while loading configuration file: #{e}")
|
74
|
-
EventHub.logger.info(
|
74
|
+
EventHub.logger.info("Using default configuration values")
|
75
75
|
end
|
76
76
|
|
77
77
|
deep_merge!(@config_data, default_configuration)
|
@@ -94,13 +94,18 @@ module EventHub
|
|
94
94
|
fail(NoMethodError, "unknown configuration [#{name}]", caller)
|
95
95
|
end
|
96
96
|
|
97
|
+
def respond_to_missing?(name, include_private = false)
|
98
|
+
return true if @config_data[name.to_sym]
|
99
|
+
false
|
100
|
+
end
|
101
|
+
|
97
102
|
def default_configuration
|
98
103
|
{
|
99
104
|
server: {
|
100
|
-
user:
|
101
|
-
password:
|
102
|
-
host:
|
103
|
-
vhost:
|
105
|
+
user: "guest",
|
106
|
+
password: "guest",
|
107
|
+
host: "localhost",
|
108
|
+
vhost: "event_hub",
|
104
109
|
port: 5672,
|
105
110
|
tls: false,
|
106
111
|
tls_cert: nil,
|