eventhub-processor 0.2.1 → 0.2.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +13 -5
- data/lib/eventhub-processor.rb +29 -26
- data/lib/eventhub/argument_parser.rb +33 -0
- data/lib/eventhub/configuration.rb +26 -26
- data/lib/eventhub/helper.rb +54 -54
- data/lib/eventhub/message.rb +128 -117
- data/lib/eventhub/multi_logger.rb +89 -89
- data/lib/eventhub/processor.rb +306 -306
- data/lib/eventhub/version.rb +1 -1
- metadata +59 -28
checksums.yaml
CHANGED
@@ -1,7 +1,15 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
|
2
|
+
!binary "U0hBMQ==":
|
3
|
+
metadata.gz: !binary |-
|
4
|
+
YjAxYjAwMjEyMzZiODlhNTIzNjI2MTQ5OWI2ZDNkYzJlM2I4MTE4MQ==
|
5
|
+
data.tar.gz: !binary |-
|
6
|
+
M2Q2NjM4YTg2YTBmMWViMzcyYTU0ODQ3NDAzZjgyYzhlNjlkYjQwYQ==
|
5
7
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
|
8
|
+
metadata.gz: !binary |-
|
9
|
+
ZGNlNGQ1MzE4NTRiYmFkMDE0MjY0ZTFlNjZiMGZlNGQzZjYxZTNkMWFhYzJl
|
10
|
+
MzUyNGVlYTkzNDYyYzc4YTE3YmQ2N2NmM2M0NjdjYjVhNGE3MmEwZTI2M2Ux
|
11
|
+
NDVmOWNiOGM2OGExYjQwN2U0NWIyZGMwYmYwZjE5ZmQ4MzQ0NjI=
|
12
|
+
data.tar.gz: !binary |-
|
13
|
+
ZGExOGJlMTVkNDA4NzVlOWRlNjRjMDVmMmUzY2M0MDcwYTNlMjRjNDBiMDFk
|
14
|
+
ZDg4OGY0NmE0YWU4OTllMjllYWQ4YjNkN2ZiMmJjNTYyNjAwZTgwMjM3NGEw
|
15
|
+
OWY4ZTBiMmYzYjFlMzQ0MzMyMWZiZTFkYWU2NmM2NzE2M2RhYjI=
|
data/lib/eventhub-processor.rb
CHANGED
@@ -1,27 +1,30 @@
|
|
1
|
-
require 'amqp'
|
2
|
-
require 'rest-client'
|
3
|
-
require 'json'
|
4
|
-
require 'singleton'
|
5
|
-
require 'uuidtools'
|
6
|
-
require 'base64'
|
7
|
-
require 'socket'
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
require_relative 'eventhub/
|
12
|
-
require_relative 'eventhub/
|
13
|
-
|
14
|
-
require_relative 'eventhub/
|
15
|
-
require_relative 'eventhub/
|
16
|
-
|
17
|
-
require_relative 'eventhub/
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
1
|
+
require 'amqp'
|
2
|
+
require 'rest-client'
|
3
|
+
require 'json'
|
4
|
+
require 'singleton'
|
5
|
+
require 'uuidtools'
|
6
|
+
require 'base64'
|
7
|
+
require 'socket'
|
8
|
+
require 'ostruct'
|
9
|
+
require 'optparse'
|
10
|
+
|
11
|
+
require_relative 'eventhub/argument_parser'
|
12
|
+
require_relative 'eventhub/version'
|
13
|
+
require_relative 'eventhub/constant'
|
14
|
+
require_relative 'eventhub/helper'
|
15
|
+
require_relative 'eventhub/multi_logger'
|
16
|
+
|
17
|
+
require_relative 'eventhub/configuration'
|
18
|
+
require_relative 'eventhub/hash_extensions'
|
19
|
+
require_relative 'eventhub/processor'
|
20
|
+
require_relative 'eventhub/message'
|
21
|
+
|
22
|
+
module EventHub
|
23
|
+
def self.logger
|
24
|
+
unless @logger
|
25
|
+
@logger = MultiLogger.new
|
26
|
+
@logger.add_device(Logger.new(STDOUT))
|
27
|
+
end
|
28
|
+
@logger
|
29
|
+
end
|
27
30
|
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
module EventHub
|
2
|
+
|
3
|
+
class ArgumentParser
|
4
|
+
def self.parse(args)
|
5
|
+
# The options specified on the command line will be collected in *options*.
|
6
|
+
# We set default values here.
|
7
|
+
options = OpenStruct.new
|
8
|
+
options.environment = 'development'
|
9
|
+
options.detached = false
|
10
|
+
|
11
|
+
opt_parser = OptionParser.new do |opts|
|
12
|
+
opts.banner = "Usage: #{args[0]}.rb [options]"
|
13
|
+
|
14
|
+
opts.on("-e", "--environment ENVIRONMENT","Environment the processor is running") do |environment|
|
15
|
+
options.environment = environment
|
16
|
+
end
|
17
|
+
|
18
|
+
opts.on("-d", "--detached", "Run processor detached as a daemon") do |v|
|
19
|
+
options.detached = v
|
20
|
+
end
|
21
|
+
|
22
|
+
opts.on_tail("-h", "--help", "Show this message") do
|
23
|
+
puts opts
|
24
|
+
exit
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
opt_parser.parse!(args)
|
29
|
+
options
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
end
|
@@ -1,27 +1,27 @@
|
|
1
|
-
module EventHub
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
include Helper
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
@folder = Dir.pwd
|
12
|
-
@environment = 'development'
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
@environment = env
|
19
|
-
true
|
20
|
-
rescue => e
|
21
|
-
EventHub.logger.info("Unexpected exception while loading configuration [#{input}]: #{format_string(e.message)}")
|
22
|
-
|
23
|
-
end
|
24
|
-
|
25
|
-
|
26
|
-
|
1
|
+
module EventHub
|
2
|
+
|
3
|
+
class Configuration
|
4
|
+
include Singleton
|
5
|
+
include Helper
|
6
|
+
|
7
|
+
attr_accessor :data, :folder, :environment
|
8
|
+
|
9
|
+
def initialize
|
10
|
+
@data = nil
|
11
|
+
@folder = Dir.pwd
|
12
|
+
@environment = 'development'
|
13
|
+
end
|
14
|
+
|
15
|
+
def load_file(input, env = 'development')
|
16
|
+
tmp = JSON.parse( IO.read(input))
|
17
|
+
@data = tmp[env]
|
18
|
+
@environment = env
|
19
|
+
true
|
20
|
+
rescue => e
|
21
|
+
EventHub.logger.info("Unexpected exception while loading configuration [#{input}]: #{format_string(e.message)}")
|
22
|
+
false
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
26
|
+
|
27
27
|
end
|
data/lib/eventhub/helper.rb
CHANGED
@@ -1,55 +1,55 @@
|
|
1
|
-
module EventHub
|
2
|
-
|
3
|
-
module Helper
|
4
|
-
|
5
|
-
# converts a class like EventHub::PlateStore::MyClassName to an array ['event_hub','plate_store','my_class_name']
|
6
|
-
def class_to_array(class_name)
|
7
|
-
class_name.to_s.split("::").map{ |m| m.gsub(/[A-Z]/) { |c| "_#{c}"}.gsub(/^_/,"").downcase }
|
8
|
-
end
|
9
|
-
|
10
|
-
# replaces CR, LF, CRLF with ";" and cut's string to requied length by adding "..." if string would be longer
|
11
|
-
def format_string(message,max_characters=80)
|
12
|
-
max_characters = 5 if max_characters < 5
|
13
|
-
m = message.gsub(/\r\n|\n|\r/m,";")
|
14
|
-
return (m[0..max_characters-4] + "...") if m.size > max_characters
|
15
|
-
return m
|
16
|
-
end
|
17
|
-
|
18
|
-
def get_host
|
19
|
-
Socket.gethostname
|
20
|
-
end
|
21
|
-
|
22
|
-
def get_ip_adresses
|
23
|
-
list = Socket.ip_address_list.map { |i| i.ip_address unless i.ipv4_loopback? || i.ipv6_loopback? }.compact
|
24
|
-
return list.size == 0 ? ["no ip address found (loopback excluded)"] : list
|
25
|
-
end
|
26
|
-
|
27
|
-
def now_stamp(now=nil)
|
28
|
-
now ||= Time.now
|
29
|
-
now.utc.strftime("%Y-%m-%dT%H:%M:%S.#{now.usec}Z")
|
30
|
-
end
|
31
|
-
|
32
|
-
def duration(difference)
|
33
|
-
rest, secs = difference.divmod( 60 ) # self is the time difference t2 - t1
|
34
|
-
rest, mins = rest.divmod( 60 )
|
35
|
-
days, hours = rest.divmod( 24 )
|
36
|
-
secs = secs.truncate
|
37
|
-
milliseconds = ((difference - difference.truncate)*1000).round
|
38
|
-
|
39
|
-
result = []
|
40
|
-
result << "#{days} days" if days > 1
|
41
|
-
result << "#{days} day" if days == 1
|
42
|
-
result << "#{hours} hours" if hours > 1
|
43
|
-
result << "#{hours} hour" if hours == 1
|
44
|
-
result << "#{mins} minutes" if mins > 1
|
45
|
-
result << "#{mins} minute" if mins == 1
|
46
|
-
result << "#{secs} seconds" if secs > 1
|
47
|
-
result << "#{secs} second" if secs == 1
|
48
|
-
result << "#{milliseconds} milliseconds" if milliseconds > 1
|
49
|
-
result << "#{milliseconds} millisecond" if milliseconds == 1
|
50
|
-
return result.join(' ')
|
51
|
-
end
|
52
|
-
|
53
|
-
end
|
54
|
-
|
1
|
+
module EventHub
|
2
|
+
|
3
|
+
module Helper
|
4
|
+
|
5
|
+
# converts a class like EventHub::PlateStore::MyClassName to an array ['event_hub','plate_store','my_class_name']
|
6
|
+
def class_to_array(class_name)
|
7
|
+
class_name.to_s.split("::").map{ |m| m.gsub(/[A-Z]/) { |c| "_#{c}"}.gsub(/^_/,"").downcase }
|
8
|
+
end
|
9
|
+
|
10
|
+
# replaces CR, LF, CRLF with ";" and cut's string to requied length by adding "..." if string would be longer
|
11
|
+
def format_string(message,max_characters=80)
|
12
|
+
max_characters = 5 if max_characters < 5
|
13
|
+
m = message.gsub(/\r\n|\n|\r/m,";")
|
14
|
+
return (m[0..max_characters-4] + "...") if m.size > max_characters
|
15
|
+
return m
|
16
|
+
end
|
17
|
+
|
18
|
+
def get_host
|
19
|
+
Socket.gethostname
|
20
|
+
end
|
21
|
+
|
22
|
+
def get_ip_adresses
|
23
|
+
list = Socket.ip_address_list.map { |i| i.ip_address unless i.ipv4_loopback? || i.ipv6_loopback? }.compact
|
24
|
+
return list.size == 0 ? ["no ip address found (loopback excluded)"] : list
|
25
|
+
end
|
26
|
+
|
27
|
+
def now_stamp(now=nil)
|
28
|
+
now ||= Time.now
|
29
|
+
now.utc.strftime("%Y-%m-%dT%H:%M:%S.#{now.usec}Z")
|
30
|
+
end
|
31
|
+
|
32
|
+
def duration(difference)
|
33
|
+
rest, secs = difference.divmod( 60 ) # self is the time difference t2 - t1
|
34
|
+
rest, mins = rest.divmod( 60 )
|
35
|
+
days, hours = rest.divmod( 24 )
|
36
|
+
secs = secs.truncate
|
37
|
+
milliseconds = ((difference - difference.truncate)*1000).round
|
38
|
+
|
39
|
+
result = []
|
40
|
+
result << "#{days} days" if days > 1
|
41
|
+
result << "#{days} day" if days == 1
|
42
|
+
result << "#{hours} hours" if hours > 1
|
43
|
+
result << "#{hours} hour" if hours == 1
|
44
|
+
result << "#{mins} minutes" if mins > 1
|
45
|
+
result << "#{mins} minute" if mins == 1
|
46
|
+
result << "#{secs} seconds" if secs > 1
|
47
|
+
result << "#{secs} second" if secs == 1
|
48
|
+
result << "#{milliseconds} milliseconds" if milliseconds > 1
|
49
|
+
result << "#{milliseconds} millisecond" if milliseconds == 1
|
50
|
+
return result.join(' ')
|
51
|
+
end
|
52
|
+
|
53
|
+
end
|
54
|
+
|
55
55
|
end
|
data/lib/eventhub/message.rb
CHANGED
@@ -1,117 +1,128 @@
|
|
1
|
-
module EventHub
|
2
|
-
|
3
|
-
class Message
|
4
|
-
include Helper
|
5
|
-
|
6
|
-
VERSION = '1.0.0'
|
7
|
-
|
8
|
-
# Headers that are required (value can be nil) in order to pass valid?
|
9
|
-
REQUIRED_HEADERS = [
|
10
|
-
'message_id',
|
11
|
-
'version',
|
12
|
-
'created_at',
|
13
|
-
'origin.module_id',
|
14
|
-
'origin.type',
|
15
|
-
'origin.site_id',
|
16
|
-
'process.name',
|
17
|
-
'process.step_position',
|
18
|
-
'process.execution_id',
|
19
|
-
'status.retried_count',
|
20
|
-
'status.code',
|
21
|
-
'status.message'
|
22
|
-
]
|
23
|
-
|
24
|
-
attr_accessor :header, :body, :raw, :vhost, :routing_key
|
25
|
-
|
26
|
-
# Build accessors for all required headers
|
27
|
-
REQUIRED_HEADERS.each do |header|
|
28
|
-
name = header.gsub(/\./,"_")
|
29
|
-
|
30
|
-
define_method(name) do
|
31
|
-
self.header.get(header)
|
32
|
-
end
|
33
|
-
|
34
|
-
define_method("#{name}=") do |value|
|
35
|
-
self.header.set(header,value)
|
36
|
-
end
|
37
|
-
end
|
38
|
-
|
39
|
-
def self.from_json(raw)
|
40
|
-
data = JSON.parse(raw)
|
41
|
-
Message.new(data.get('header'), data.get('body'),raw)
|
42
|
-
rescue => e
|
43
|
-
Message.new({ "status" => { "code" => STATUS_INVALID, "message" => "JSON parse error: #{e}" }} ,{ "original_message_base64_encoded" => Base64.encode64(raw)},raw)
|
44
|
-
end
|
45
|
-
|
46
|
-
# process_step_position should be
|
47
|
-
def initialize(header=nil, body=nil,raw=nil)
|
48
|
-
|
49
|
-
@header = header || {}
|
50
|
-
@body = body || {}
|
51
|
-
@raw = raw
|
52
|
-
|
53
|
-
# set message defaults, that we have required headers
|
54
|
-
@header.set('message_id',UUIDTools::UUID.timestamp_create.to_s,false)
|
55
|
-
@header.set('version',VERSION,false)
|
56
|
-
@header.set('created_at',now_stamp,false)
|
57
|
-
|
58
|
-
@header.set('origin.module_id','undefined',false)
|
59
|
-
@header.set('origin.type','undefined',false)
|
60
|
-
@header.set('origin.site_id','undefined',false)
|
61
|
-
|
62
|
-
@header.set('process.name','undefined',false)
|
63
|
-
@header.set('process.execution_id',UUIDTools::UUID.timestamp_create.to_s,false)
|
64
|
-
@header.set('process.step_position',0,false)
|
65
|
-
|
66
|
-
@header.set('status.retried_count',0,false)
|
67
|
-
@header.set('status.code',STATUS_INITIAL,false)
|
68
|
-
@header.set('status.message','',false)
|
69
|
-
|
70
|
-
end
|
71
|
-
|
72
|
-
def valid?
|
73
|
-
# check for existence and defined value
|
74
|
-
REQUIRED_HEADERS.all? { |key| @header.all_keys_with_path.include?(key) && !!self.send(key.gsub(/\./,"_").to_sym)}
|
75
|
-
end
|
76
|
-
|
77
|
-
def success?
|
78
|
-
self.status_code == STATUS_SUCCESS
|
79
|
-
end
|
80
|
-
|
81
|
-
def retry?
|
82
|
-
!success?
|
83
|
-
end
|
84
|
-
|
85
|
-
def initial?
|
86
|
-
self.status_code == STATUS_INITIAL
|
87
|
-
end
|
88
|
-
|
89
|
-
def retry_pending?
|
90
|
-
self.status_code == STATUS_RETRY_PENDING
|
91
|
-
end
|
92
|
-
|
93
|
-
def to_json
|
94
|
-
{'header' => self.header, 'body' => self.body}.to_json
|
95
|
-
end
|
96
|
-
|
97
|
-
def to_s
|
98
|
-
"Msg: process [#{self.process_name},#{self.process_step_position},#{self.process_execution_id}], status [#{self.status_code},#{self.status_message},#{self.status_retried_count}]"
|
99
|
-
end
|
100
|
-
|
101
|
-
# copies the message and set's provided status code (default: success), actual stamp, and a new message id
|
102
|
-
def copy(status_code=STATUS_SUCCESS)
|
103
|
-
|
104
|
-
# use Marshal dump and load to make a deep object copy
|
105
|
-
copied_header = Marshal.load( Marshal.dump(header))
|
106
|
-
copied_body = Marshal.load( Marshal.dump(body))
|
107
|
-
|
108
|
-
copied_header.set("message_id",UUIDTools::UUID.timestamp_create.to_s)
|
109
|
-
copied_header.set("created_at",now_stamp)
|
110
|
-
copied_header.set("status.code",status_code)
|
111
|
-
|
112
|
-
Message.new(copied_header, copied_body)
|
113
|
-
end
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
1
|
+
module EventHub
|
2
|
+
|
3
|
+
class Message
|
4
|
+
include Helper
|
5
|
+
|
6
|
+
VERSION = '1.0.0'
|
7
|
+
|
8
|
+
# Headers that are required (value can be nil) in order to pass valid?
|
9
|
+
REQUIRED_HEADERS = [
|
10
|
+
'message_id',
|
11
|
+
'version',
|
12
|
+
'created_at',
|
13
|
+
'origin.module_id',
|
14
|
+
'origin.type',
|
15
|
+
'origin.site_id',
|
16
|
+
'process.name',
|
17
|
+
'process.step_position',
|
18
|
+
'process.execution_id',
|
19
|
+
'status.retried_count',
|
20
|
+
'status.code',
|
21
|
+
'status.message'
|
22
|
+
]
|
23
|
+
|
24
|
+
attr_accessor :header, :body, :raw, :vhost, :routing_key
|
25
|
+
|
26
|
+
# Build accessors for all required headers
|
27
|
+
REQUIRED_HEADERS.each do |header|
|
28
|
+
name = header.gsub(/\./,"_")
|
29
|
+
|
30
|
+
define_method(name) do
|
31
|
+
self.header.get(header)
|
32
|
+
end
|
33
|
+
|
34
|
+
define_method("#{name}=") do |value|
|
35
|
+
self.header.set(header,value)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def self.from_json(raw)
|
40
|
+
data = JSON.parse(raw)
|
41
|
+
Message.new(data.get('header'), data.get('body'),raw)
|
42
|
+
rescue => e
|
43
|
+
Message.new({ "status" => { "code" => STATUS_INVALID, "message" => "JSON parse error: #{e}" }} ,{ "original_message_base64_encoded" => Base64.encode64(raw)},raw)
|
44
|
+
end
|
45
|
+
|
46
|
+
# process_step_position should be
|
47
|
+
def initialize(header=nil, body=nil,raw=nil)
|
48
|
+
|
49
|
+
@header = header || {}
|
50
|
+
@body = body || {}
|
51
|
+
@raw = raw
|
52
|
+
|
53
|
+
# set message defaults, that we have required headers
|
54
|
+
@header.set('message_id',UUIDTools::UUID.timestamp_create.to_s,false)
|
55
|
+
@header.set('version',VERSION,false)
|
56
|
+
@header.set('created_at',now_stamp,false)
|
57
|
+
|
58
|
+
@header.set('origin.module_id','undefined',false)
|
59
|
+
@header.set('origin.type','undefined',false)
|
60
|
+
@header.set('origin.site_id','undefined',false)
|
61
|
+
|
62
|
+
@header.set('process.name','undefined',false)
|
63
|
+
@header.set('process.execution_id',UUIDTools::UUID.timestamp_create.to_s,false)
|
64
|
+
@header.set('process.step_position',0,false)
|
65
|
+
|
66
|
+
@header.set('status.retried_count',0,false)
|
67
|
+
@header.set('status.code',STATUS_INITIAL,false)
|
68
|
+
@header.set('status.message','',false)
|
69
|
+
|
70
|
+
end
|
71
|
+
|
72
|
+
def valid?
|
73
|
+
# check for existence and defined value
|
74
|
+
REQUIRED_HEADERS.all? { |key| @header.all_keys_with_path.include?(key) && !!self.send(key.gsub(/\./,"_").to_sym)}
|
75
|
+
end
|
76
|
+
|
77
|
+
def success?
|
78
|
+
self.status_code == STATUS_SUCCESS
|
79
|
+
end
|
80
|
+
|
81
|
+
def retry?
|
82
|
+
!success?
|
83
|
+
end
|
84
|
+
|
85
|
+
def initial?
|
86
|
+
self.status_code == STATUS_INITIAL
|
87
|
+
end
|
88
|
+
|
89
|
+
def retry_pending?
|
90
|
+
self.status_code == STATUS_RETRY_PENDING
|
91
|
+
end
|
92
|
+
|
93
|
+
def to_json
|
94
|
+
{'header' => self.header, 'body' => self.body}.to_json
|
95
|
+
end
|
96
|
+
|
97
|
+
def to_s
|
98
|
+
"Msg: process [#{self.process_name},#{self.process_step_position},#{self.process_execution_id}], status [#{self.status_code},#{self.status_message},#{self.status_retried_count}]"
|
99
|
+
end
|
100
|
+
|
101
|
+
# copies the message and set's provided status code (default: success), actual stamp, and a new message id
|
102
|
+
def copy(status_code=STATUS_SUCCESS)
|
103
|
+
|
104
|
+
# use Marshal dump and load to make a deep object copy
|
105
|
+
copied_header = Marshal.load( Marshal.dump(header))
|
106
|
+
copied_body = Marshal.load( Marshal.dump(body))
|
107
|
+
|
108
|
+
copied_header.set("message_id",UUIDTools::UUID.timestamp_create.to_s)
|
109
|
+
copied_header.set("created_at",now_stamp)
|
110
|
+
copied_header.set("status.code",status_code)
|
111
|
+
|
112
|
+
Message.new(copied_header, copied_body)
|
113
|
+
end
|
114
|
+
|
115
|
+
def self.translate_status_code(code)
|
116
|
+
case code
|
117
|
+
when EventHub::STATUS_INITIAL then return 'STATUS_INITIAL'
|
118
|
+
when EventHub::STATUS_SUCCESS then return 'STATUS_SUCCESS'
|
119
|
+
when EventHub::STATUS_RETRY then return 'STATUS_RETRY'
|
120
|
+
when EventHub::STATUS_RETRY_PENDING then return 'STATUS_RETRY_PENDING'
|
121
|
+
when EventHub::STATUS_INVALID then return 'STATUS_INVALID'
|
122
|
+
when EventHub::STATUS_DEADLETTER then return 'STATUS_DEADLETTER'
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
end
|
127
|
+
|
128
|
+
end
|