eventhub-processor2 1.8.0 → 1.9.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rspec +1 -0
- data/.ruby-version +1 -1
- data/.travis.yml +2 -2
- data/CHANGELOG.md +6 -0
- data/Gemfile +1 -1
- data/Gemfile.lock +57 -41
- data/Rakefile +7 -4
- data/docker/Dockerfile +1 -1
- data/docker/docker-compose.yml +1 -1
- data/eventhub-processor2.gemspec +24 -23
- data/example/crasher.rb +12 -13
- data/example/publisher.rb +41 -41
- data/example/receiver.rb +2 -2
- data/example/router.rb +3 -3
- data/lib/eventhub/actor_heartbeat.rb +23 -23
- data/lib/eventhub/actor_listener.rb +17 -21
- data/lib/eventhub/actor_publisher.rb +7 -7
- data/lib/eventhub/actor_watchdog.rb +3 -4
- data/lib/eventhub/base.rb +24 -31
- data/lib/eventhub/base_exception.rb +2 -2
- data/lib/eventhub/configuration.rb +30 -25
- data/lib/eventhub/constant.rb +24 -24
- data/lib/eventhub/hash_extensions.rb +8 -8
- data/lib/eventhub/helper.rb +7 -7
- data/lib/eventhub/logger.rb +2 -2
- data/lib/eventhub/message.rb +45 -45
- data/lib/eventhub/processor2.rb +22 -22
- data/lib/eventhub/sleeper.rb +1 -1
- data/lib/eventhub/statistics.rb +6 -8
- data/lib/eventhub/version.rb +1 -1
- metadata +24 -10
data/example/receiver.rb
CHANGED
@@ -1,9 +1,9 @@
|
|
1
|
-
require_relative
|
1
|
+
require_relative "../lib/eventhub/base"
|
2
2
|
|
3
3
|
module EventHub
|
4
4
|
class Receiver < Processor2
|
5
5
|
def handle_message(message, args = {})
|
6
|
-
id = message.body[
|
6
|
+
id = message.body["id"]
|
7
7
|
EventHub.logger.info("[#{id}] - Received")
|
8
8
|
|
9
9
|
file_name = "data/#{id}.json"
|
data/example/router.rb
CHANGED
@@ -1,12 +1,12 @@
|
|
1
|
-
require_relative
|
1
|
+
require_relative "../lib/eventhub/base"
|
2
2
|
|
3
3
|
module EventHub
|
4
4
|
# Demo class
|
5
5
|
class Router < Processor2
|
6
6
|
def handle_message(message, args = {})
|
7
|
-
id = message.body[
|
7
|
+
id = message.body["id"]
|
8
8
|
EventHub.logger.info("Received: [#{id}]")
|
9
|
-
publish(message: message.to_json, exchange_name:
|
9
|
+
publish(message: message.to_json, exchange_name: "example.inbound")
|
10
10
|
EventHub.logger.info("Returned: [#{id}]")
|
11
11
|
nil
|
12
12
|
end
|
@@ -12,23 +12,23 @@ module EventHub
|
|
12
12
|
end
|
13
13
|
|
14
14
|
def start
|
15
|
-
EventHub.logger.info(
|
15
|
+
EventHub.logger.info("Heartbeat is starting...")
|
16
16
|
|
17
|
-
every(
|
17
|
+
every(60 * 60 * 24) { EventHub.logger.info("Actual actors: #{Celluloid::Actor.all.size}: #{Celluloid::Actor.all.map { |a| a.class }.join(", ")}") }
|
18
18
|
|
19
|
-
publish(heartbeat(action:
|
20
|
-
EventHub.logger.info(
|
19
|
+
publish(heartbeat(action: "started"))
|
20
|
+
EventHub.logger.info("Heartbeat has sent [started] beat")
|
21
21
|
loop do
|
22
22
|
sleep Configuration.processor[:heartbeat_cycle_in_s]
|
23
23
|
publish(heartbeat)
|
24
|
-
EventHub.logger.info(
|
24
|
+
EventHub.logger.info("Heartbeat has sent a beat")
|
25
25
|
end
|
26
26
|
end
|
27
27
|
|
28
28
|
def cleanup
|
29
|
-
EventHub.logger.info(
|
30
|
-
publish(heartbeat(action:
|
31
|
-
EventHub.logger.info(
|
29
|
+
EventHub.logger.info("Heartbeat is cleaning up...")
|
30
|
+
publish(heartbeat(action: "stopped"))
|
31
|
+
EventHub.logger.info("Heartbeat has sent a [stopped] beat")
|
32
32
|
end
|
33
33
|
|
34
34
|
private
|
@@ -43,29 +43,29 @@ module EventHub
|
|
43
43
|
success = channel.wait_for_confirms
|
44
44
|
|
45
45
|
unless success
|
46
|
-
raise
|
47
|
-
|
46
|
+
raise "Published heartbeat message has "\
|
47
|
+
"not been confirmed by the server"
|
48
48
|
end
|
49
49
|
ensure
|
50
|
-
connection
|
50
|
+
connection&.close
|
51
51
|
end
|
52
52
|
|
53
|
-
def heartbeat(args = {
|
53
|
+
def heartbeat(args = {action: "running"})
|
54
54
|
message = EventHub::Message.new
|
55
|
-
message.origin_module_id
|
56
|
-
message.origin_type
|
57
|
-
message.origin_site_id
|
55
|
+
message.origin_module_id = EventHub::Configuration.name
|
56
|
+
message.origin_type = "processor"
|
57
|
+
message.origin_site_id = "global"
|
58
58
|
|
59
|
-
message.process_name
|
59
|
+
message.process_name = "event_hub.heartbeat"
|
60
60
|
|
61
61
|
now = Time.now
|
62
62
|
|
63
63
|
# message structure needs more changes
|
64
64
|
message.body = {
|
65
65
|
version: @processor_instance.send(:version),
|
66
|
-
action:
|
67
|
-
pid:
|
68
|
-
process_name:
|
66
|
+
action: args[:action],
|
67
|
+
pid: Process.pid,
|
68
|
+
process_name: "event_hub.heartbeat",
|
69
69
|
heartbeat: {
|
70
70
|
started: now_stamp(started_at),
|
71
71
|
stamp_last_beat: now_stamp(now),
|
@@ -90,11 +90,11 @@ module EventHub
|
|
90
90
|
end
|
91
91
|
|
92
92
|
def addresses
|
93
|
-
interfaces = Socket.getifaddrs.select
|
93
|
+
interfaces = Socket.getifaddrs.select { |interface|
|
94
94
|
!interface.addr.ipv4_loopback? && !interface.addr.ipv6_loopback?
|
95
|
-
|
95
|
+
}
|
96
96
|
|
97
|
-
interfaces.map
|
97
|
+
interfaces.map { |interface|
|
98
98
|
begin
|
99
99
|
{
|
100
100
|
interface: interface.name,
|
@@ -104,7 +104,7 @@ module EventHub
|
|
104
104
|
rescue
|
105
105
|
nil # will be ignored
|
106
106
|
end
|
107
|
-
|
107
|
+
}.compact
|
108
108
|
end
|
109
109
|
|
110
110
|
def messages_statistics
|
@@ -15,14 +15,14 @@ module EventHub
|
|
15
15
|
end
|
16
16
|
|
17
17
|
def start
|
18
|
-
EventHub.logger.info(
|
18
|
+
EventHub.logger.info("Listener is starting...")
|
19
19
|
EventHub::Configuration.processor[:listener_queues].each_with_index do |queue_name, index|
|
20
20
|
async.listen(queue_name: queue_name, index: index)
|
21
21
|
end
|
22
22
|
end
|
23
23
|
|
24
24
|
def restart
|
25
|
-
raise
|
25
|
+
raise "Listener is restarting..."
|
26
26
|
end
|
27
27
|
|
28
28
|
def listen(args = {})
|
@@ -30,7 +30,7 @@ module EventHub
|
|
30
30
|
EventHub.logger.info("Listening to queue [#{queue_name}]")
|
31
31
|
consumer.on_delivery do |delivery_info, metadata, payload|
|
32
32
|
EventHub.logger.info("#{queue_name}: [#{delivery_info.delivery_tag}]"\
|
33
|
-
|
33
|
+
" delivery")
|
34
34
|
|
35
35
|
@processor_instance.statistics.measure(payload.size) do
|
36
36
|
handle_payload(payload: payload,
|
@@ -38,25 +38,21 @@ module EventHub
|
|
38
38
|
queue_name: queue_name,
|
39
39
|
content_type: metadata[:content_type],
|
40
40
|
priority: metadata[:priority],
|
41
|
-
delivery_tag: delivery_info.delivery_tag
|
42
|
-
)
|
41
|
+
delivery_tag: delivery_info.delivery_tag)
|
43
42
|
channel.acknowledge(delivery_info.delivery_tag, false)
|
44
43
|
end
|
45
44
|
|
46
45
|
EventHub.logger.info("#{queue_name}: [#{delivery_info.delivery_tag}]"\
|
47
|
-
|
46
|
+
" acknowledged")
|
48
47
|
end
|
49
48
|
queue.subscribe_with(consumer, block: false)
|
50
49
|
end
|
51
|
-
|
52
50
|
rescue => error
|
53
51
|
EventHub.logger.error("Unexpected exception: #{error}. It should restart now with this exception...")
|
54
52
|
raise
|
55
53
|
end
|
56
54
|
|
57
55
|
def with_listen(args = {}, &block)
|
58
|
-
connection_string, connection_properties = connection_properties
|
59
|
-
|
60
56
|
connection = create_bunny_connection
|
61
57
|
connection.start
|
62
58
|
queue_name = args[:queue_name]
|
@@ -65,11 +61,11 @@ module EventHub
|
|
65
61
|
channel.prefetch(1)
|
66
62
|
queue = channel.queue(queue_name, durable: true)
|
67
63
|
consumer = EventHub::Consumer.new(channel,
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
64
|
+
queue,
|
65
|
+
EventHub::Configuration.name +
|
66
|
+
"-" +
|
67
|
+
args[:index].to_s,
|
68
|
+
false)
|
73
69
|
yield connection, channel, consumer, queue, queue_name
|
74
70
|
end
|
75
71
|
|
@@ -86,18 +82,18 @@ module EventHub
|
|
86
82
|
# return invalid messages to dispatcher
|
87
83
|
if message.invalid?
|
88
84
|
response_messages << message
|
89
|
-
EventHub.logger.info("-> #{message
|
85
|
+
EventHub.logger.info("-> #{message} => return invalid to dispatcher")
|
90
86
|
else
|
91
87
|
begin
|
92
88
|
response_messages = @processor_instance.send(:handle_message,
|
93
|
-
|
94
|
-
|
89
|
+
message,
|
90
|
+
pass_arguments(args))
|
95
91
|
rescue => exception
|
96
92
|
# this catches unexpected exceptions in handle message method
|
97
93
|
# deadletter the message via dispatcher
|
98
94
|
message.status_code = EventHub::STATUS_DEADLETTER
|
99
95
|
message.status_message = exception.to_s
|
100
|
-
EventHub.logger.info("-> #{message
|
96
|
+
EventHub.logger.info("-> #{message} => return exception to dispatcher")
|
101
97
|
response_messages << message
|
102
98
|
end
|
103
99
|
end
|
@@ -109,15 +105,15 @@ module EventHub
|
|
109
105
|
|
110
106
|
def pass_arguments(args = {})
|
111
107
|
keys_to_pass = [:queue_name, :content_type, :priority, :delivery_tag]
|
112
|
-
args.select{ |key| keys_to_pass.include?(key) }
|
108
|
+
args.select { |key| keys_to_pass.include?(key) }
|
113
109
|
end
|
114
110
|
|
115
111
|
def cleanup
|
116
|
-
EventHub.logger.info(
|
112
|
+
EventHub.logger.info("Listener is cleaning up...")
|
117
113
|
# close all open connections
|
118
114
|
return unless @connections
|
119
115
|
@connections.values.each do |connection|
|
120
|
-
connection
|
116
|
+
connection&.close
|
121
117
|
end
|
122
118
|
end
|
123
119
|
|
@@ -7,7 +7,7 @@ module EventHub
|
|
7
7
|
finalizer :cleanup
|
8
8
|
|
9
9
|
def initialize
|
10
|
-
EventHub.logger.info(
|
10
|
+
EventHub.logger.info("Publisher is starting...")
|
11
11
|
@connection = nil
|
12
12
|
end
|
13
13
|
|
@@ -31,16 +31,16 @@ module EventHub
|
|
31
31
|
success = channel.wait_for_confirms
|
32
32
|
|
33
33
|
unless success
|
34
|
-
raise
|
35
|
-
|
34
|
+
raise "Published message from Listener actor "\
|
35
|
+
"has not been confirmed by the server"
|
36
36
|
end
|
37
|
-
|
38
|
-
|
37
|
+
ensure
|
38
|
+
channel&.close
|
39
39
|
end
|
40
40
|
|
41
41
|
def cleanup
|
42
|
-
EventHub.logger.info(
|
43
|
-
@connection
|
42
|
+
EventHub.logger.info("Publisher is cleaning up...")
|
43
|
+
@connection&.close
|
44
44
|
end
|
45
45
|
end
|
46
46
|
end
|
@@ -7,20 +7,19 @@ module EventHub
|
|
7
7
|
finalizer :cleanup
|
8
8
|
|
9
9
|
def initialize
|
10
|
-
EventHub.logger.info(
|
10
|
+
EventHub.logger.info("Watchdog is starting...")
|
11
11
|
async.start
|
12
12
|
end
|
13
13
|
|
14
14
|
def start
|
15
15
|
loop do
|
16
|
-
EventHub.logger.info('Running watchdog...')
|
17
16
|
watch
|
18
17
|
sleep Configuration.processor[:watchdog_cycle_in_s]
|
19
18
|
end
|
20
19
|
end
|
21
20
|
|
22
21
|
def cleanup
|
23
|
-
EventHub.logger.info(
|
22
|
+
EventHub.logger.info("Watchdog is cleaning up...")
|
24
23
|
end
|
25
24
|
|
26
25
|
private
|
@@ -36,7 +35,7 @@ module EventHub
|
|
36
35
|
end
|
37
36
|
end
|
38
37
|
ensure
|
39
|
-
connection
|
38
|
+
connection&.close
|
40
39
|
end
|
41
40
|
end
|
42
41
|
end
|
data/lib/eventhub/base.rb
CHANGED
@@ -1,36 +1,29 @@
|
|
1
|
-
require
|
2
|
-
require
|
3
|
-
require
|
4
|
-
require
|
1
|
+
require "uuidtools"
|
2
|
+
require "json"
|
3
|
+
require "base64"
|
4
|
+
require "optparse"
|
5
5
|
|
6
|
-
require
|
7
|
-
require
|
8
|
-
require
|
6
|
+
require "eventhub/components"
|
7
|
+
require "logstash-logger"
|
8
|
+
require "bunny"
|
9
|
+
require "celluloid"
|
9
10
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
require_relative
|
19
|
-
require_relative
|
20
|
-
require_relative
|
21
|
-
require_relative
|
22
|
-
require_relative
|
23
|
-
require_relative
|
24
|
-
require_relative
|
25
|
-
require_relative
|
26
|
-
require_relative 'message'
|
27
|
-
require_relative 'statistics'
|
28
|
-
require_relative 'consumer'
|
29
|
-
require_relative 'actor_heartbeat'
|
30
|
-
require_relative 'actor_watchdog'
|
31
|
-
require_relative 'actor_publisher'
|
32
|
-
require_relative 'actor_listener'
|
33
|
-
require_relative 'processor2'
|
11
|
+
require_relative "version"
|
12
|
+
require_relative "constant"
|
13
|
+
require_relative "base_exception"
|
14
|
+
require_relative "logger"
|
15
|
+
require_relative "helper"
|
16
|
+
require_relative "sleeper"
|
17
|
+
require_relative "hash_extensions"
|
18
|
+
require_relative "configuration"
|
19
|
+
require_relative "message"
|
20
|
+
require_relative "statistics"
|
21
|
+
require_relative "consumer"
|
22
|
+
require_relative "actor_heartbeat"
|
23
|
+
require_relative "actor_watchdog"
|
24
|
+
require_relative "actor_publisher"
|
25
|
+
require_relative "actor_listener"
|
26
|
+
require_relative "processor2"
|
34
27
|
|
35
28
|
Celluloid.logger = nil
|
36
29
|
Celluloid.exception_handler { |ex| EventHub.logger.error "Exception occured: #{ex}" }
|
@@ -3,9 +3,9 @@ module EventHub
|
|
3
3
|
# BaseException class
|
4
4
|
class BaseException < RuntimeError
|
5
5
|
attr_accessor :code, :message
|
6
|
-
def initialize(message=nil, code=EventHub::STATUS_DEADLETTER)
|
6
|
+
def initialize(message = nil, code = EventHub::STATUS_DEADLETTER)
|
7
7
|
@message = message
|
8
|
-
@code
|
8
|
+
@code = code
|
9
9
|
super(@message)
|
10
10
|
end
|
11
11
|
end
|
@@ -6,16 +6,16 @@ module EventHub
|
|
6
6
|
extend self
|
7
7
|
extend Helper
|
8
8
|
|
9
|
-
attr_reader :name
|
10
|
-
attr_reader :environment
|
11
|
-
attr_reader :detached
|
12
|
-
attr_reader :config_file
|
13
|
-
attr_reader :config_data
|
14
|
-
|
15
|
-
@name =
|
16
|
-
@environment =
|
9
|
+
attr_reader :name # name of processor
|
10
|
+
attr_reader :environment # environment the processor is running
|
11
|
+
attr_reader :detached # run processor run as a daemon
|
12
|
+
attr_reader :config_file # name of configuration file
|
13
|
+
attr_reader :config_data # data from configuration file
|
14
|
+
|
15
|
+
@name = "undefined"
|
16
|
+
@environment = "development"
|
17
17
|
@detached = false
|
18
|
-
@config_file = File.join(Dir.getwd,
|
18
|
+
@config_file = File.join(Dir.getwd, "config", "#{@name}.json")
|
19
19
|
@config_data = {}
|
20
20
|
|
21
21
|
# set name of processor
|
@@ -24,32 +24,32 @@ module EventHub
|
|
24
24
|
end
|
25
25
|
|
26
26
|
def reset
|
27
|
-
@name =
|
28
|
-
@environment =
|
27
|
+
@name = "undefined"
|
28
|
+
@environment = "development"
|
29
29
|
@detached = false
|
30
|
-
@config_file = File.join(Dir.getwd,
|
30
|
+
@config_file = File.join(Dir.getwd, "config", "#{@name}.json")
|
31
31
|
@config_data = {}
|
32
32
|
end
|
33
33
|
|
34
34
|
# parse options from argument list
|
35
35
|
def parse_options(argv = ARGV)
|
36
|
-
@config_file = File.join(Dir.getwd,
|
36
|
+
@config_file = File.join(Dir.getwd, "config", "#{@name}.json")
|
37
37
|
|
38
|
-
OptionParser.new
|
39
|
-
note =
|
40
|
-
opts.on(
|
38
|
+
OptionParser.new { |opts|
|
39
|
+
note = "Define environment"
|
40
|
+
opts.on("-e", "--environment ENVIRONMENT", note) do |environment|
|
41
41
|
@environment = environment
|
42
42
|
end
|
43
43
|
|
44
|
-
opts.on(
|
44
|
+
opts.on("-d", "--detached", "Run processor detached as a daemon") do
|
45
45
|
@detached = true
|
46
46
|
end
|
47
47
|
|
48
|
-
note =
|
49
|
-
opts.on(
|
48
|
+
note = "Define configuration file"
|
49
|
+
opts.on("-c", "--config CONFIG", note) do |config|
|
50
50
|
@config_file = config
|
51
51
|
end
|
52
|
-
|
52
|
+
}.parse!(argv)
|
53
53
|
|
54
54
|
true
|
55
55
|
rescue OptionParser::InvalidOption => e
|
@@ -71,7 +71,7 @@ module EventHub
|
|
71
71
|
new_data = JSON.parse(File.read(@config_file), symbolize_names: true)
|
72
72
|
rescue => e
|
73
73
|
EventHub.logger.warn("Exception while loading configuration file: #{e}")
|
74
|
-
EventHub.logger.info(
|
74
|
+
EventHub.logger.info("Using default configuration values")
|
75
75
|
end
|
76
76
|
|
77
77
|
deep_merge!(@config_data, default_configuration)
|
@@ -94,13 +94,18 @@ module EventHub
|
|
94
94
|
fail(NoMethodError, "unknown configuration [#{name}]", caller)
|
95
95
|
end
|
96
96
|
|
97
|
+
def respond_to_missing?(name, include_private = false)
|
98
|
+
return true if @config_data[name.to_sym]
|
99
|
+
false
|
100
|
+
end
|
101
|
+
|
97
102
|
def default_configuration
|
98
103
|
{
|
99
104
|
server: {
|
100
|
-
user:
|
101
|
-
password:
|
102
|
-
host:
|
103
|
-
vhost:
|
105
|
+
user: "guest",
|
106
|
+
password: "guest",
|
107
|
+
host: "localhost",
|
108
|
+
vhost: "event_hub",
|
104
109
|
port: 5672,
|
105
110
|
tls: false,
|
106
111
|
tls_cert: nil,
|