eventhub-processor 0.0.5 → 0.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 134f2e6d20850ddc0901916ee045823c68e10e2c
4
- data.tar.gz: f65baa4cad21c4d98fdf4a75ffb5bcf085b18e1d
3
+ metadata.gz: dea7fe8b1b08a77c68d736c1c2e2e72325df218a
4
+ data.tar.gz: bc476eac97e284575a684ed0cfe61850d4ea9b8f
5
5
  SHA512:
6
- metadata.gz: 34501530c76eb1150b4291e9ea4eff0f874e25965447678ead5ec629d0117cb32f2de45e45310b59238392dd7cf82dbe9619e0cd4edae884db89a39723d2d2ec
7
- data.tar.gz: b7a2958916947dfa8ef261a0d862e7783ebb7832f43ca6d24f0d78d6139a523fedebd40616efc5e6dc05edcb5ea025ea63f67cccb282b0b0a1e492b0a75ca9c6
6
+ metadata.gz: dcce6e241aefab6b054f79d942969722fa04ea9162e86fbcefce05e396d39c69062785ae0a2eca6409dd4e0864d604bbf84092f977f80d7a7068b531cc0c044b
7
+ data.tar.gz: be828faa80aaa01901a29918413f265956835686b5120402726eec913eded92203fe57760f1b4f744da63ebbc9f307efe22175c4382662ab6a4322c0d9ee5d09
@@ -4,9 +4,11 @@ require 'rest-client'
4
4
  require 'json'
5
5
  require 'singleton'
6
6
  require 'uuidtools'
7
+ require 'base64'
7
8
 
8
9
  require_relative 'eventhub/version'
9
10
  require_relative 'eventhub/constant'
11
+ require_relative 'eventhub/helper'
10
12
  require_relative 'eventhub/multi_logger'
11
13
 
12
14
  require_relative 'eventhub/configuration'
@@ -2,6 +2,7 @@ module EventHub
2
2
 
3
3
  class Configuration
4
4
  include Singleton
5
+ include Helper
5
6
 
6
7
  attr_accessor :data
7
8
 
@@ -12,7 +13,11 @@ module EventHub
12
13
  def load_file(input, env='development')
13
14
  tmp = JSON.parse( IO.read(input))
14
15
  @data = tmp[env]
15
- end
16
+ true
17
+ rescue => e
18
+ EventHub.logger.info("Unexpected exception while loading configuration [#{input}]: #{format_raw_string(e.message)}")
19
+ false
20
+ end
16
21
 
17
22
  end
18
23
 
@@ -0,0 +1,19 @@
1
+ module EventHub
2
+
3
+ module Helper
4
+
5
+ # converts a base class name, Whatever::MyClassName => my_class_name
6
+ def class_to_string(class_name)
7
+ class_name.to_s.split("::")[-1].gsub(/[A-Z]/) { |c| "_#{c}"}.gsub(/^_/,"").downcase
8
+ end
9
+
10
+ def format_raw_string(message,max_characters=80)
11
+ max_characters = 5 if max_characters < 5
12
+ m = message.gsub(/\r\n|\n|\r/m,";")
13
+ return (m[0..max_characters-3] + "...") if m.size > max_characters
14
+ return m
15
+ end
16
+
17
+ end
18
+
19
+ end
@@ -35,15 +35,15 @@ module EventHub
35
35
  end
36
36
  end
37
37
 
38
- def self.from_json(json)
39
- data = JSON.parse(json)
40
- Message.new(data.get('header'), data.get('body'),json)
38
+ def self.from_json(raw)
39
+ data = JSON.parse(raw)
40
+ Message.new(data.get('header'), data.get('body'),raw)
41
41
  rescue => e
42
- Message.new({ "status" => { "code" => STATUS_INVALID, "message" => "JSON parse error: #{e}" }} ,{},json)
42
+ Message.new({ "status" => { "code" => STATUS_INVALID, "message" => "JSON parse error: #{e}" }} ,{ "original_message_base64_encoded" => Base64.encode64(raw)},raw)
43
43
  end
44
44
 
45
45
  # process_step_position should be
46
- def initialize(header, body,raw=nil)
46
+ def initialize(header=nil, body=nil,raw=nil)
47
47
 
48
48
  @header = header || {}
49
49
  @body = body || {}
@@ -15,8 +15,16 @@ end
15
15
  module EventHub
16
16
 
17
17
  class MultiLogger
18
- def initialize(*targets)
19
- @targets = targets
18
+
19
+ attr_accessor :folder, :devices
20
+
21
+ def initialize(folder=nil)
22
+ @folder = folder || Dir.pwd
23
+ @devices = []
24
+ end
25
+
26
+ def add_device(device)
27
+ @devices << device
20
28
  end
21
29
 
22
30
  def save_detailed_error(feedback,message=nil)
@@ -24,9 +32,9 @@ module EventHub
24
32
  stamp = "#{time.strftime("%Y%m%d_%H%M%S")}_#{"%03d" % (time.usec/1000)}"
25
33
  filename = "#{stamp}.log"
26
34
 
27
- FileUtils.makedirs("exceptions")
35
+ FileUtils.makedirs("#{folder}/exceptions")
28
36
 
29
- File.open("exceptions/#{filename}","w") do |output|
37
+ File.open("#{@folder}/exceptions/#{filename}","w") do |output|
30
38
  output.write("#{feedback}\n\n")
31
39
  output.write("Exception: #{feedback.class.to_s}\n\n")
32
40
  output.write("Call Stack:\n")
@@ -37,7 +45,7 @@ module EventHub
37
45
 
38
46
  # save message if provided
39
47
  if message
40
- File.open("exceptions/#{stamp}.msg.raw","wb") do |output|
48
+ File.open("#{@folder}/exceptions/#{stamp}.msg.raw","wb") do |output|
41
49
  output.write(message)
42
50
  end
43
51
  end
@@ -47,7 +55,7 @@ module EventHub
47
55
 
48
56
  %w(log debug info warn error).each do |m|
49
57
  define_method(m) do |*args|
50
- @targets.map { |t| t.send(m, *args) }
58
+ @devices.map { |d| d.send(m, *args) }
51
59
  end
52
60
  end
53
61
 
@@ -1,6 +1,26 @@
1
1
  module EventHub
2
2
  class Processor
3
3
 
4
+ attr_accessor :name, :folder
5
+
6
+ include Helper
7
+
8
+ def initialize(name=nil)
9
+ @name = name || class_to_string(self.class)
10
+ @folder = Dir.pwd
11
+
12
+ @started = Time.now
13
+
14
+ @messages_successful = 0
15
+ @messages_unsuccessful = 0
16
+
17
+ @restart = true
18
+ end
19
+
20
+ def configuration
21
+ EventHub::Configuration.instance.data
22
+ end
23
+
4
24
  def host
5
25
  configuration.get('server.host') || 'localhost'
6
26
  end
@@ -17,27 +37,30 @@ module EventHub
17
37
  configuration.get('server.management_port') || 15672
18
38
  end
19
39
 
20
- def queue_name
21
- configuration.get('processor.queue') || 'inbound'
40
+ def listener_queue
41
+ configuration.get('processor.listener_queue') || 'inbound'
22
42
  end
23
43
 
24
44
  def vhost
25
45
  configuration.get('server.vhost') || nil
26
46
  end
27
47
 
28
- def watchdog_cycle
29
- configuration.get('processor.watchdog_cycle') || 5
48
+ def watchdog_cycle_in_s
49
+ configuration.get('processor.watchdog_cycle_is_s') || 15
30
50
  end
31
51
 
32
- def configuration
33
- EventHub::Configuration.instance.data
52
+ def restart_in_s
53
+ configuration.get('processor.restart_in_s') || 15
34
54
  end
35
55
 
56
+ def heartbeat_cycle_in_s
57
+ configuration.get('processor.heartbeat_cycle_in_s') || 300
58
+ end
36
59
 
60
+ def start(detached=false)
61
+ daemonize if detached
37
62
 
38
- def start
39
- restart = true
40
- while restart
63
+ while @restart
41
64
 
42
65
  begin
43
66
  AMQP.start(configuration.get('server')) do |connection, open_ok|
@@ -46,9 +69,8 @@ module EventHub
46
69
 
47
70
  # deal with tcp connection issues
48
71
  @connection.on_tcp_connection_loss do |conn, settings|
49
- EventHub.logger.warn("Processor lost tcp connection. Trying to restart in 5 seconds...")
50
- sleep 5
51
- EventMachine.stop
72
+ EventHub.logger.warn("Processor lost tcp connection. Trying to restart in #{@restart_in_s} seconds...")
73
+ stop_processor(true)
52
74
  end
53
75
 
54
76
  # create channel
@@ -56,75 +78,108 @@ module EventHub
56
78
  @channel.auto_recovery = true
57
79
 
58
80
  # connect to queue
59
- @queue = @channel.queue(queue_name, durable: true, auto_delete: false)
81
+ @queue = @channel.queue(self.listener_queue, durable: true, auto_delete: false)
60
82
 
61
83
  # subscribe to queue
62
84
  @queue.subscribe(:ack => true) do |metadata, payload|
63
- handle_heartbeat(payload)
64
- if handle_message(metadata,payload)
65
- metadata.ack
85
+ begin
86
+ if handle_message(metadata,payload)
87
+ raise
88
+ metadata.ack
89
+ else
90
+ metadata.nack
91
+ end
92
+ rescue => e
93
+ EventHub.logger.error("Unexpected exception in handle_message method: #{e}")
94
+ EventHub.logger.save_detailed_error(e)
66
95
  end
67
96
  end
68
97
 
69
- # Features to stop main event loop
70
- stop_main_loop = Proc.new {
71
- @connection.disconnect {
72
- EventHub.logger.info("Processor is stopping main event loop")
73
- EventMachine.stop
74
- restart = false
75
- }
76
- }
98
+ EventHub.logger.info("Processor [#{@name}] is listening to queue [#{self.listener_queue}], base folder [#{@folder}]")
77
99
 
78
- Signal.trap "TERM", stop_main_loop
79
- Signal.trap "INT", stop_main_loop
100
+ # Singnal Listening
101
+ Signal.trap("TERM") {stop_processor}
102
+ Signal.trap("INT") {stop_processor}
80
103
 
81
- EventMachine.add_timer(self.watchdog_cycle) { watchdog }
104
+ # Various timers
105
+ EventMachine.add_timer(@watchdog_cycle_in_s) { watchdog }
82
106
 
83
- EventHub.logger.info("Processor is listening to queue [#{[configuration.get('server.vhost'),configuration.get('processor.queue')].compact.join(".")}]")
107
+ heartbeat
84
108
  end
85
109
  rescue => e
86
- EventHub.logger.error("Unexpected exception: #{e}")
87
- EventHub.logger.save_detailed_error(e)
110
+ Signal.trap("TERM") { stop_processor }
111
+ Signal.trap("INT") { stop_processor }
112
+
113
+ id = EventHub.logger.save_detailed_error(e)
114
+ EventHub.logger.error("Unexpected exception: #{e}, see => #{id}")
115
+
116
+ sleep_break self.restart_in_s
88
117
  end
89
118
 
90
119
  end # while
91
120
 
92
- EventHub.logger.info("Processor has been stopped")
121
+ EventHub.logger.info("Processor [#{@name}] has been stopped")
122
+ ensure
123
+ # remove pid file
124
+ begin
125
+ File.delete("#{@folder}/#{@name}.pid")
126
+ rescue
127
+ # ignore exceptions here
128
+ end
93
129
  end
94
130
 
95
131
  def handle_message(metadata,payload)
96
132
  raise "Please implement method in derived class"
97
133
  end
98
134
 
99
- def handle_heartbeat(message)
100
- # sends a standard message back to dispatcher
101
- end
102
-
103
135
  def watchdog
104
136
  begin
105
- response = RestClient.get "http://#{self.user}:#{self.password}@#{host}:#{management_port}/api/queues/#{self.vhost}/#{self.queue_name}/bindings", { :content_type => :json}
137
+ response = RestClient.get "http://#{self.user}:#{self.password}@#{self.host}:#{self.management_port}/api/queues/#{self.vhost}/#{self.listener_queue}/bindings", { :content_type => :json}
106
138
  data = JSON.parse(response.body)
107
139
 
108
140
  if response.code != 200
109
- EventHub.logger.warn("Watchdog: Server did not answered properly. Trying to restart in 5 seconds...")
110
- sleep 5
111
- EventMachine.stop
141
+ EventHub.logger.warn("Watchdog: Server did not answered properly. Trying to restart in #{self.restart_in_s} seconds...")
142
+ stop_processor
112
143
  elsif data.size == 0
113
- EventHub.logger.warn("Watchdog: Something is wrong with the vhost, queue, and/or bindings. Trying to restart in 5 seconds...")
114
- sleep 5
115
- EventMachine.stop
144
+ EventHub.logger.warn("Watchdog: Something is wrong with the vhost, queue, and/or bindings. Trying to restart in #{self.restart_in_s} seconds...")
145
+ stop_processor
116
146
  else
117
147
  # Watchdog is happy :-)
118
- end
119
-
148
+ # add timer for next check
149
+ EventMachine.add_timer(self.watchdog_cycle_in_s) { watchdog }
150
+ end
151
+
120
152
  rescue => e
121
- EventHub.logger.error("Watchdog: Unexpected exception: #{e}. Trying to restart in 5 seconds...")
122
- sleep 5
123
- EventMachine.stop
153
+ EventHub.logger.error("Watchdog: Unexpected exception: #{e}. Trying to restart in #{self.restart_in_s} seconds...")
154
+ stop_processor
124
155
  end
156
+ end
125
157
 
126
- # place next time
127
- EventMachine.add_timer(watchdog_cycle) { watchdog }
158
+ def heartbeat
159
+ message = Message.new
160
+ message.origin_module_id = @name
161
+ message.origin_type = "processor"
162
+ message.origin_site_id = 'chbs'
163
+
164
+ message.process_name = 'event_hub.heartbeat'
165
+
166
+ message.body = {
167
+ heartbeat: {
168
+ started: @started,
169
+ stamp_last_beat: Time.now,
170
+ heartbeat_cycle_in_s: self.heartbeat_cycle_in_s,
171
+ served_queues: [self.listener_queue],
172
+ messages: {
173
+ total: @messages_successful+@messages_unsuccessful,
174
+ successful: @messages_successful,
175
+ unsuccessful: @messages_unsuccessful
176
+ }
177
+ }
178
+ }
179
+
180
+ send_to_dispatcher(message.to_json)
181
+
182
+ EventMachine.add_timer(self.heartbeat_cycle_in_s) { heartbeat }
128
183
  end
129
184
 
130
185
  def send_to_dispatcher(payload)
@@ -136,12 +191,15 @@ module EventHub
136
191
  channel = connection.create_channel
137
192
  channel.confirm_select
138
193
 
139
- channel.default_exchange.publish(payload,routing_key: 'inbound', persistent: true)
194
+ channel.default_exchange.publish(payload,routing_key: EVENT_HUB_QUEUE_INBOUND, persistent: true)
140
195
  success = channel.wait_for_confirms
141
196
 
142
197
  if !success
143
198
  EventHub.logger.error("Message has not been confirmed by the server to be received !!!")
144
199
  confirmed = false
200
+ @messages_unsuccessful += 1
201
+ else
202
+ @messages_successful += 1
145
203
  end
146
204
 
147
205
  channel.close
@@ -150,12 +208,35 @@ module EventHub
150
208
  confirmed
151
209
  end
152
210
 
153
- def format_raw_string(message,max_characters=80)
154
- max_characters = 5 if max_characters < 5
155
- m = message.gsub(/\r\n/m,";")
156
- return (m[0..max_characters-3] + "...") if m.size > max_characters
157
- return m
158
- end
211
+ def sleep_break( seconds ) # breaks after n seconds or after interrupt
212
+ while (seconds > 0)
213
+ sleep(1)
214
+ seconds -= 1
215
+ break unless @restart
216
+ end
217
+ end
218
+
219
+ private
220
+
221
+ def stop_processor(restart=false)
222
+ @restart = restart
223
+
224
+ # stop event loop
225
+ @connection.disconnect {
226
+ EventHub.logger.info("Processor [#{@name}] is stopping main event loop")
227
+ EventMachine.stop
228
+ }
229
+ end
230
+
231
+ def daemonize
232
+ EventHub.logger.info("Processor [#{@name}] is going to start as daemon")
233
+
234
+ # daemonize
235
+ Process.daemon
236
+
237
+ # write daemon pid
238
+ IO.write("#{@folder}/#{@name}.pid",Process.pid.to_s)
239
+ end
159
240
 
160
241
  end
161
242
  end
@@ -1,3 +1,3 @@
1
1
  module EventHub
2
- VERSION = "0.0.5"
2
+ VERSION = "0.0.6"
3
3
  end
metadata CHANGED
@@ -1,97 +1,97 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: eventhub-processor
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.5
4
+ version: 0.0.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - Thomas Steiner
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-04-01 00:00:00.000000000 Z
11
+ date: 2014-04-03 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
- - - ~>
17
+ - - "~>"
18
18
  - !ruby/object:Gem::Version
19
19
  version: '1.3'
20
20
  type: :development
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
- - - ~>
24
+ - - "~>"
25
25
  - !ruby/object:Gem::Version
26
26
  version: '1.3'
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: rake
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
- - - '>='
31
+ - - ">="
32
32
  - !ruby/object:Gem::Version
33
33
  version: '0'
34
34
  type: :development
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
- - - '>='
38
+ - - ">="
39
39
  - !ruby/object:Gem::Version
40
40
  version: '0'
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: rest-client
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
- - - '>='
45
+ - - ">="
46
46
  - !ruby/object:Gem::Version
47
47
  version: '0'
48
48
  type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
- - - '>='
52
+ - - ">="
53
53
  - !ruby/object:Gem::Version
54
54
  version: '0'
55
55
  - !ruby/object:Gem::Dependency
56
56
  name: amqp
57
57
  requirement: !ruby/object:Gem::Requirement
58
58
  requirements:
59
- - - '>='
59
+ - - ">="
60
60
  - !ruby/object:Gem::Version
61
61
  version: '0'
62
62
  type: :runtime
63
63
  prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
- - - '>='
66
+ - - ">="
67
67
  - !ruby/object:Gem::Version
68
68
  version: '0'
69
69
  - !ruby/object:Gem::Dependency
70
70
  name: bunny
71
71
  requirement: !ruby/object:Gem::Requirement
72
72
  requirements:
73
- - - '>='
73
+ - - ">="
74
74
  - !ruby/object:Gem::Version
75
75
  version: '0'
76
76
  type: :runtime
77
77
  prerelease: false
78
78
  version_requirements: !ruby/object:Gem::Requirement
79
79
  requirements:
80
- - - '>='
80
+ - - ">="
81
81
  - !ruby/object:Gem::Version
82
82
  version: '0'
83
83
  - !ruby/object:Gem::Dependency
84
84
  name: uuidtools
85
85
  requirement: !ruby/object:Gem::Requirement
86
86
  requirements:
87
- - - '>='
87
+ - - ">="
88
88
  - !ruby/object:Gem::Version
89
89
  version: '0'
90
90
  type: :runtime
91
91
  prerelease: false
92
92
  version_requirements: !ruby/object:Gem::Requirement
93
93
  requirements:
94
- - - '>='
94
+ - - ">="
95
95
  - !ruby/object:Gem::Version
96
96
  version: '0'
97
97
  description: Gem to build Event Hub processors
@@ -101,16 +101,17 @@ executables: []
101
101
  extensions: []
102
102
  extra_rdoc_files: []
103
103
  files:
104
+ - LICENSE.txt
105
+ - Rakefile
106
+ - lib/eventhub-processor.rb
104
107
  - lib/eventhub/configuration.rb
105
108
  - lib/eventhub/constant.rb
106
109
  - lib/eventhub/hash.rb
110
+ - lib/eventhub/helper.rb
107
111
  - lib/eventhub/message.rb
108
112
  - lib/eventhub/multi_logger.rb
109
113
  - lib/eventhub/processor.rb
110
114
  - lib/eventhub/version.rb
111
- - lib/eventhub-processor.rb
112
- - LICENSE.txt
113
- - Rakefile
114
115
  homepage: http://github.com/thomis/eventhub-processor
115
116
  licenses:
116
117
  - MIT
@@ -121,17 +122,17 @@ require_paths:
121
122
  - lib
122
123
  required_ruby_version: !ruby/object:Gem::Requirement
123
124
  requirements:
124
- - - '>='
125
+ - - ">="
125
126
  - !ruby/object:Gem::Version
126
127
  version: '0'
127
128
  required_rubygems_version: !ruby/object:Gem::Requirement
128
129
  requirements:
129
- - - '>='
130
+ - - ">="
130
131
  - !ruby/object:Gem::Version
131
132
  version: '0'
132
133
  requirements: []
133
134
  rubyforge_project:
134
- rubygems_version: 2.0.3
135
+ rubygems_version: 2.2.2
135
136
  signing_key:
136
137
  specification_version: 4
137
138
  summary: Gem to build Event Hub processors