eventhub-processor 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: d49b2fd5c92b9661ed09bd39b1121c38cd761dd0
4
- data.tar.gz: c3d0641193fd5dec17225c2291d8795f8e00542f
3
+ metadata.gz: 7aa5e7a8e2f31b9683c5ff655c76c1c46bc75372
4
+ data.tar.gz: 717a76158ee4c97ba9b5595a8d21814bd36d0102
5
5
  SHA512:
6
- metadata.gz: cd0bffe7bcd37d0201528368a394c1bb8171e866841589b60d87f75360c9e3eab616848ca9d441cb801c7d5648c6397d910b0fe694e3bcd89c4524aae6e4c302
7
- data.tar.gz: 9e06375d46cde18f2ebb0397d4f1ccdf0bbbd9757b7a3d2e9583330b49ab8e8b58cd295fcbb5d0e05abdfeb608c26c2fac425083d5cb3912eade85b7817a3f8f
6
+ metadata.gz: ba810bf60a874be8e0611d2156e17a90e67a12dbfe68258d17b11ddcfdeb490fd237b8426322328d46b6bea5d818f5078c29bb5b944e3534d1f1b964dea72104
7
+ data.tar.gz: 335f306fd2c7300967fd702dace1239eca7a07b4030ee1b0be50b3f723c627704c15027e944d7c159bddac96b0e4cf440cf9574b8637f901c4f1a8a11fc81cb9
@@ -1,37 +1,37 @@
1
- require 'amqp'
2
- require 'rest-client'
3
- require 'json'
4
- require 'singleton'
5
- require 'uuidtools'
6
- require 'base64'
7
- require 'socket'
8
- require 'ostruct'
9
- require 'optparse'
10
-
11
- require_relative 'eventhub/argument_parser'
12
- require_relative 'eventhub/version'
13
- require_relative 'eventhub/constant'
14
- require_relative 'eventhub/helper'
15
- require_relative 'eventhub/multi_logger'
16
-
17
- require_relative 'eventhub/configuration'
18
- require_relative 'eventhub/hash_extensions'
19
- require_relative 'eventhub/statistics'
20
- require_relative 'eventhub/heartbeat'
21
- require_relative 'eventhub/pidfile'
22
- require_relative 'eventhub/processor'
23
- require_relative 'eventhub/message_processor'
24
- require_relative 'eventhub/message'
25
-
26
- require_relative 'eventhub/base_exception'
27
- require_relative 'eventhub/no_deadletter_exception'
28
-
29
- module EventHub
30
- def self.logger
31
- unless @logger
32
- @logger = MultiLogger.new
33
- @logger.add_device(Logger.new(STDOUT))
34
- end
35
- @logger
36
- end
37
- end
1
+ require 'amqp'
2
+ require 'rest-client'
3
+ require 'json'
4
+ require 'singleton'
5
+ require 'uuidtools'
6
+ require 'base64'
7
+ require 'socket'
8
+ require 'ostruct'
9
+ require 'optparse'
10
+
11
+ require_relative 'eventhub/argument_parser'
12
+ require_relative 'eventhub/version'
13
+ require_relative 'eventhub/constant'
14
+ require_relative 'eventhub/helper'
15
+ require_relative 'eventhub/multi_logger'
16
+
17
+ require_relative 'eventhub/configuration'
18
+ require_relative 'eventhub/hash_extensions'
19
+ require_relative 'eventhub/statistics'
20
+ require_relative 'eventhub/heartbeat'
21
+ require_relative 'eventhub/pidfile'
22
+ require_relative 'eventhub/processor'
23
+ require_relative 'eventhub/message_processor'
24
+ require_relative 'eventhub/message'
25
+
26
+ require_relative 'eventhub/base_exception'
27
+ require_relative 'eventhub/no_deadletter_exception'
28
+
29
+ module EventHub
30
+ def self.logger
31
+ unless @logger
32
+ @logger = ::EventHub::Components::MultiLogger.new
33
+ @logger.add_device(Logger.new(STDOUT))
34
+ end
35
+ @logger
36
+ end
37
+ end
@@ -1,35 +1,35 @@
1
- module EventHub
2
-
3
- class ArgumentParser
4
-
5
- def self.parse(args)
6
- # The options specified on the command line will be collected in *options*.
7
- # We set default values here.
8
- options = OpenStruct.new
9
- options.environment = 'development'
10
- options.detached = false
11
-
12
- opt_parser = OptionParser.new do |opts|
13
- opts.banner = "Usage: #{args[0]}.rb [options]"
14
- yield(opts, options) if block_given? # allow to add more options
15
-
16
- opts.on("-e", "--environment ENVIRONMENT","Environment the processor is running") do |environment|
17
- options.environment = environment
18
- end
19
-
20
- opts.on("-d", "--detached", "Run processor detached as a daemon") do |v|
21
- options.detached = v
22
- end
23
-
24
- opts.on_tail("-h", "--help", "Show this message") do
25
- puts opts
26
- exit
27
- end
28
- end
29
-
30
- opt_parser.parse!(args)
31
- options
32
- end
33
- end
34
-
1
+ module EventHub
2
+
3
+ class ArgumentParser
4
+
5
+ def self.parse(args)
6
+ # The options specified on the command line will be collected in *options*.
7
+ # We set default values here.
8
+ options = OpenStruct.new
9
+ options.environment = 'development'
10
+ options.detached = false
11
+
12
+ opt_parser = OptionParser.new do |opts|
13
+ opts.banner = "Usage: #{args[0]}.rb [options]"
14
+ yield(opts, options) if block_given? # allow to add more options
15
+
16
+ opts.on("-e", "--environment ENVIRONMENT","Environment the processor is running") do |environment|
17
+ options.environment = environment
18
+ end
19
+
20
+ opts.on("-d", "--detached", "Run processor detached as a daemon") do |v|
21
+ options.detached = v
22
+ end
23
+
24
+ opts.on_tail("-h", "--help", "Show this message") do
25
+ puts opts
26
+ exit
27
+ end
28
+ end
29
+
30
+ opt_parser.parse!(args)
31
+ options
32
+ end
33
+ end
34
+
35
35
  end
@@ -1,2 +1,2 @@
1
- class EventHub::BaseException < RuntimeError
1
+ class EventHub::BaseException < RuntimeError
2
2
  end
@@ -1,26 +1,26 @@
1
- module EventHub
2
-
3
- class Configuration
4
- include Singleton
5
- include Helper
6
-
7
- attr_accessor :data, :folder, :environment
8
-
9
- def initialize
10
- @data = nil
11
- @environment = 'development'
12
- end
13
-
14
- def load_file(input, env = 'development')
15
- json = JSON.parse(IO.read(input))
16
- @data = json[env]
17
- @environment = env
18
- true
19
- rescue => e
20
- EventHub.logger.info("Unexpected exception while loading configuration [#{input}]: #{format_string(e.message)}")
21
- false
22
- end
23
-
24
- end
25
-
1
+ module EventHub
2
+
3
+ class Configuration
4
+ include Singleton
5
+ include Helper
6
+
7
+ attr_accessor :data, :folder, :environment
8
+
9
+ def initialize
10
+ @data = nil
11
+ @environment = 'development'
12
+ end
13
+
14
+ def load_file(input, env = 'development')
15
+ json = JSON.parse(IO.read(input))
16
+ @data = json[env]
17
+ @environment = env
18
+ true
19
+ rescue => e
20
+ EventHub.logger.info("Unexpected exception while loading configuration [#{input}]: #{format_string(e.message)}")
21
+ false
22
+ end
23
+
24
+ end
25
+
26
26
  end
@@ -1,16 +1,16 @@
1
- module EventHub
2
-
3
- EH_X_INBOUND = 'event_hub.inbound'
4
-
5
- STATUS_INITIAL = 0 # To be set when dispatcher needs to dispatch to first process step.
6
- STATUS_SUCCESS = 200 # To be set to indicate successful processed message. Dispatcher will routes message to the next step.
7
- STATUS_RETRY = 300 # To be set to trigger retry cycle controlled by the dispatcher
8
- STATUS_RETRY_PENDING = 301 # Set and used by the dispatcher only.
9
- # Set before putting the message into a retry queue.
10
- # Once message has been retried it will sent do the same step with status.code = STATUS_SUCCESS
11
- STATUS_INVALID = 400 # To be set to indicate invalid message (not json, invalid Event Hub Message).
12
- # Dispatcher will publish message to the invalid queue.
13
- STATUS_DEADLETTER = 500 # To be set by dispatcher, processor or channel adapters to indicate
14
- # that message needs to be dead-lettered. Rejected messages could miss the
15
- # status.code = STATUS_DEADLETTER due to the RabbitMQ deadletter exchange mechanism.
1
+ module EventHub
2
+
3
+ EH_X_INBOUND = 'event_hub.inbound'
4
+
5
+ STATUS_INITIAL = 0 # To be set when dispatcher needs to dispatch to first process step.
6
+ STATUS_SUCCESS = 200 # To be set to indicate successful processed message. Dispatcher will routes message to the next step.
7
+ STATUS_RETRY = 300 # To be set to trigger retry cycle controlled by the dispatcher
8
+ STATUS_RETRY_PENDING = 301 # Set and used by the dispatcher only.
9
+ # Set before putting the message into a retry queue.
10
+ # Once message has been retried it will sent do the same step with status.code = STATUS_SUCCESS
11
+ STATUS_INVALID = 400 # To be set to indicate invalid message (not json, invalid Event Hub Message).
12
+ # Dispatcher will publish message to the invalid queue.
13
+ STATUS_DEADLETTER = 500 # To be set by dispatcher, processor or channel adapters to indicate
14
+ # that message needs to be dead-lettered. Rejected messages could miss the
15
+ # status.code = STATUS_DEADLETTER due to the RabbitMQ deadletter exchange mechanism.
16
16
  end
@@ -1,48 +1,48 @@
1
- module HashExtensions
2
-
3
- module ClassMethods
4
- end
5
-
6
- module InstanceMethods
7
- # get value from provided key path, e.g. hash.get(%w(event_hub plate.queue1 retry_s))
8
- # "a" => { "b" => { "c" => { "value"}}}
9
- def get(arg)
10
- path = arg.is_a?(String) ? arg.split('.') : arg
11
- path.inject(self,:[])
12
- rescue NoMethodError
13
- return nil
14
- end
15
-
16
- # set value from provided key path, e.h. hash.set('a.b.c','new value')
17
- # if overwrite is false, value will be set if it was nil previously
18
- def set(arg,value,overwrite=true)
19
- *key_path, last = arg.is_a?(String) ? arg.split(".") : arg
20
- if overwrite
21
- key_path.inject(self) { |h,key| h.has_key?(key) ? h[key] : h[key]={}} [last] = value
22
- else
23
- key_path.inject(self) { |h,key| h.has_key?(key) ? h[key] : h[key]={}} [last] ||= value
24
- end
25
- end
26
-
27
- # get all keys path, { 'a' => 'value1', 'b' => { 'c' => 'value2'}}.all_keys_with_path => ['a','b.c']
28
- def all_keys_with_path(parent=nil)
29
- a = []
30
- each do |k,v|
31
- if v.is_a?(Hash)
32
- a << v.all_keys_with_path([parent,k].compact.join('.'))
33
- else
34
- a << "#{[parent,k].compact.join(".")}"
35
- end
36
- end
37
- a.flatten
38
- end
39
-
40
- end
41
-
42
- def self.included(receiver)
43
- receiver.extend ClassMethods
44
- receiver.send :include, InstanceMethods
45
- end
46
- end
47
-
48
- HashExtensions.included(Hash)
1
+ module HashExtensions
2
+
3
+ module ClassMethods
4
+ end
5
+
6
+ module InstanceMethods
7
+ # get value from provided key path, e.g. hash.get(%w(event_hub plate.queue1 retry_s))
8
+ # "a" => { "b" => { "c" => { "value"}}}
9
+ def get(arg)
10
+ path = arg.is_a?(String) ? arg.split('.') : arg
11
+ path.inject(self,:[])
12
+ rescue NoMethodError
13
+ return nil
14
+ end
15
+
16
+ # set value from provided key path, e.h. hash.set('a.b.c','new value')
17
+ # if overwrite is false, value will be set if it was nil previously
18
+ def set(arg,value,overwrite=true)
19
+ *key_path, last = arg.is_a?(String) ? arg.split(".") : arg
20
+ if overwrite
21
+ key_path.inject(self) { |h,key| h.has_key?(key) ? h[key] : h[key]={}} [last] = value
22
+ else
23
+ key_path.inject(self) { |h,key| h.has_key?(key) ? h[key] : h[key]={}} [last] ||= value
24
+ end
25
+ end
26
+
27
+ # get all keys path, { 'a' => 'value1', 'b' => { 'c' => 'value2'}}.all_keys_with_path => ['a','b.c']
28
+ def all_keys_with_path(parent=nil)
29
+ a = []
30
+ each do |k,v|
31
+ if v.is_a?(Hash)
32
+ a << v.all_keys_with_path([parent,k].compact.join('.'))
33
+ else
34
+ a << "#{[parent,k].compact.join(".")}"
35
+ end
36
+ end
37
+ a.flatten
38
+ end
39
+
40
+ end
41
+
42
+ def self.included(receiver)
43
+ receiver.extend ClassMethods
44
+ receiver.send :include, InstanceMethods
45
+ end
46
+ end
47
+
48
+ HashExtensions.included(Hash)
@@ -1,77 +1,77 @@
1
- module EventHub
2
-
3
- class Heartbeat
4
- include Helper
5
-
6
- attr_reader :processor, :statistics, :started_at
7
-
8
- def initialize(processor)
9
- @started_at = Time.now
10
- @processor = processor
11
- @statistics = @processor.statistics
12
- end
13
-
14
-
15
- def build_message(action = "running")
16
- message = ::EventHub::Message.new
17
- message.origin_module_id = processor.name
18
- message.origin_type = "processor"
19
- message.origin_site_id = 'global'
20
-
21
- message.process_name = 'event_hub.heartbeat'
22
-
23
- now = Time.now
24
-
25
- # message structure needs more changes
26
- message.body = {
27
- version: processor.version,
28
- action: action,
29
- pid: Process.pid,
30
- process_name: 'event_hub.heartbeat',
31
-
32
- heartbeat: {
33
- started: now_stamp(started_at),
34
- stamp_last_beat: now_stamp(now),
35
- uptime_in_ms: (now - started_at)*1000,
36
- heartbeat_cycle_in_ms: processor.heartbeat_cycle_in_s * 1000,
37
- queues_consuming_from: processor.listener_queues,
38
- queues_publishing_to: ['event_hub.inbound'], # needs more dynamic in the future
39
- host: Socket.gethostname,
40
- addresses: addresses,
41
- messages: {
42
- total: statistics.messages_total,
43
- successful: statistics.messages_successful,
44
- unsuccessful: statistics.messages_unsuccessful,
45
- average_size: statistics.messages_average_size,
46
- average_process_time_in_ms: statistics.messages_average_process_time*1000,
47
- total_process_time_in_ms: statistics.messages_total_process_time*1000
48
- }
49
- }
50
- }
51
- message
52
- end
53
-
54
- private
55
-
56
- def addresses
57
- interfaces = Socket.getifaddrs.select do |interface|
58
- !interface.addr.ipv4_loopback? && !interface.addr.ipv6_loopback?
59
- end
60
-
61
- interfaces.map do |interface|
62
- begin
63
- {
64
- :interface => interface.name,
65
- :host_name => Socket.gethostname,
66
- :ip_address => interface.addr.ip_address
67
- }
68
- rescue
69
- nil # will be ignored
70
- end
71
- end.compact
72
-
73
- end
74
-
75
- end
76
-
77
- end
1
+ module EventHub
2
+
3
+ class Heartbeat
4
+ include Helper
5
+
6
+ attr_reader :processor, :statistics, :started_at
7
+
8
+ def initialize(processor)
9
+ @started_at = Time.now
10
+ @processor = processor
11
+ @statistics = @processor.statistics
12
+ end
13
+
14
+
15
+ def build_message(action = "running")
16
+ message = ::EventHub::Message.new
17
+ message.origin_module_id = processor.name
18
+ message.origin_type = "processor"
19
+ message.origin_site_id = 'global'
20
+
21
+ message.process_name = 'event_hub.heartbeat'
22
+
23
+ now = Time.now
24
+
25
+ # message structure needs more changes
26
+ message.body = {
27
+ version: processor.version,
28
+ action: action,
29
+ pid: Process.pid,
30
+ process_name: 'event_hub.heartbeat',
31
+
32
+ heartbeat: {
33
+ started: now_stamp(started_at),
34
+ stamp_last_beat: now_stamp(now),
35
+ uptime_in_ms: (now - started_at)*1000,
36
+ heartbeat_cycle_in_ms: processor.heartbeat_cycle_in_s * 1000,
37
+ queues_consuming_from: processor.listener_queues,
38
+ queues_publishing_to: ['event_hub.inbound'], # needs more dynamic in the future
39
+ host: Socket.gethostname,
40
+ addresses: addresses,
41
+ messages: {
42
+ total: statistics.messages_total,
43
+ successful: statistics.messages_successful,
44
+ unsuccessful: statistics.messages_unsuccessful,
45
+ average_size: statistics.messages_average_size,
46
+ average_process_time_in_ms: statistics.messages_average_process_time*1000,
47
+ total_process_time_in_ms: statistics.messages_total_process_time*1000
48
+ }
49
+ }
50
+ }
51
+ message
52
+ end
53
+
54
+ private
55
+
56
+ def addresses
57
+ interfaces = Socket.getifaddrs.select do |interface|
58
+ !interface.addr.ipv4_loopback? && !interface.addr.ipv6_loopback?
59
+ end
60
+
61
+ interfaces.map do |interface|
62
+ begin
63
+ {
64
+ :interface => interface.name,
65
+ :host_name => Socket.gethostname,
66
+ :ip_address => interface.addr.ip_address
67
+ }
68
+ rescue
69
+ nil # will be ignored
70
+ end
71
+ end.compact
72
+
73
+ end
74
+
75
+ end
76
+
77
+ end