gilmour 0.2.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,178 @@
1
+ # encoding: utf-8
2
+ # This is required to check whether Mash class already exists
3
+ require 'logger'
4
+ require 'securerandom'
5
+ require 'json'
6
+ require 'mash' unless class_exists? 'Mash'
7
+ require 'eventmachine'
8
+
9
+ require_relative 'protocol'
10
+ require_relative 'responder'
11
+ require_relative 'backends/backend'
12
+
13
+ # The Gilmour module
14
+ module Gilmour
15
+ LoggerLevels = {
16
+ unknown: Logger::UNKNOWN,
17
+ fatal: Logger::FATAL,
18
+ error: Logger::ERROR,
19
+ warn: Logger::WARN,
20
+ info: Logger::INFO,
21
+ debug: Logger::DEBUG
22
+ }
23
+
24
+
25
+ GLogger = Logger.new(STDERR)
26
+ EnvLoglevel = ENV["LOG_LEVEL"] ? ENV["LOG_LEVEL"].to_sym : :warn
27
+ GLogger.level = LoggerLevels[EnvLoglevel] || Logger::DEBUG
28
+
29
+ RUNNING = false
30
+ # This is the base module that should be included into the
31
+ # container class
32
+ module Base
33
+ def self.included(base)
34
+ base.extend(Registrar)
35
+ end
36
+
37
+ ######### Registration module ###########
38
+ # This module helps act as a Resistrar for subclasses
39
+ module Registrar
40
+ attr_accessor :subscribers_path
41
+ attr_accessor :backend
42
+ DEFAULT_SUBSCRIBER_PATH = 'subscribers'
43
+ @@subscribers = {} # rubocop:disable all
44
+ @@registered_services = []
45
+
46
+ # :nodoc:
47
+ def inherited(child)
48
+ @@registered_services << child
49
+ end
50
+
51
+
52
+ # Returns the subscriber classes registered
53
+ def registered_subscribers
54
+ @@registered_services
55
+ end
56
+
57
+ # Adds a listener for the given topic
58
+ # topic:: The topic to listen to
59
+ # opts: Hash of optional arguments.
60
+ # Supported options are:
61
+ #
62
+ # excl:: If true, this listener is added to a group of listeners
63
+ # with the same name as the name of the class in which this method
64
+ # is called. A message sent to the _topic_ will be processed by at
65
+ # most one listener from a group
66
+ #
67
+ # timeout: Maximum duration (seconds) that a subscriber has to
68
+ # finish the task. If the execution exceeds the timeout, gilmour
69
+ # responds with status {code:409, data: nil}
70
+ #
71
+ def listen_to(topic, opts={})
72
+ handler = Proc.new
73
+
74
+ opt_defaults = {
75
+ exclusive: false,
76
+ timeout: 600,
77
+ fork: false
78
+ }.merge(opts)
79
+
80
+ #Make sure these are not overriden by opts.
81
+ opt_defaults[:handler] = handler
82
+ opt_defaults[:subscriber] = self
83
+
84
+ @@subscribers[topic] ||= []
85
+ @@subscribers[topic] << opt_defaults
86
+ end
87
+
88
+ # Returns the list of subscribers for _topic_ or all subscribers if it is nil
89
+ def subscribers(topic = nil)
90
+ if topic
91
+ @@subscribers[topic]
92
+ else
93
+ @@subscribers
94
+ end
95
+ end
96
+
97
+ # Loads all ruby source files inside _dir_ as subscribers
98
+ # Should only be used inside the parent container class
99
+ def load_all(dir = nil)
100
+ dir ||= (subscribers_path || DEFAULT_SUBSCRIBER_PATH)
101
+ Dir["#{dir}/*.rb"].each { |f| require f }
102
+ end
103
+
104
+ # Loads the ruby file at _path_ as a subscriber
105
+ # Should only be used inside the parent container class
106
+ def load_subscriber(path)
107
+ require path
108
+ end
109
+ end
110
+
111
+ # :nodoc:
112
+ def registered_subscribers
113
+ self.class.registered_subscribers
114
+ end
115
+ ############ End Register ###############
116
+
117
+ class << self
118
+ attr_accessor :backend
119
+ end
120
+ attr_reader :backends
121
+
122
+ # Enable and return the given backend
123
+ # Should only be used inside the parent container class
124
+ # If +opts[:multi_process]+ is true, every request handler will
125
+ # be run inside a new child process.
126
+ def enable_backend(name, opts = {})
127
+ Gilmour::Backend.load_backend(name)
128
+ @backends ||= {}
129
+ @backends[name] ||= Gilmour::Backend.get(name).new(opts)
130
+ end
131
+ alias_method :get_backend, :enable_backend
132
+
133
+ def exit!
134
+ subs_by_backend = subs_grouped_by_backend
135
+ subs_by_backend.each do |b, subs|
136
+ backend = get_backend(b)
137
+ backend.setup_subscribers(subs)
138
+ if backend.report_health?
139
+ backend.unregister_health_check
140
+ end
141
+ end
142
+ end
143
+
144
+ # Starts all the listeners
145
+ # If _startloop_ is true, this method will start it's own
146
+ # event loop and not return till Eventmachine reactor is stopped
147
+ def start(startloop = false)
148
+ subs_by_backend = subs_grouped_by_backend
149
+ subs_by_backend.each do |b, subs|
150
+ backend = get_backend(b)
151
+ backend.setup_subscribers(subs)
152
+
153
+ if backend.report_health?
154
+ backend.register_health_check
155
+ end
156
+ end
157
+
158
+ if startloop
159
+ GLogger.debug 'Joining EM event loop'
160
+ EM.reactor_thread.join
161
+ end
162
+ end
163
+
164
+ private
165
+
166
+ def subs_grouped_by_backend
167
+ subs_by_backend = {}
168
+ self.class.subscribers.each do |topic, subs|
169
+ subs.each do |sub|
170
+ subs_by_backend[sub[:subscriber].backend] ||= {}
171
+ subs_by_backend[sub[:subscriber].backend][topic] ||= []
172
+ subs_by_backend[sub[:subscriber].backend][topic] << sub
173
+ end
174
+ end
175
+ subs_by_backend
176
+ end
177
+ end
178
+ end
@@ -0,0 +1,48 @@
1
+ # encoding: utf-8
2
+ require 'json'
3
+ # Top level Gilmour module
4
+ module Gilmour
5
+ # This module implements the JSON communication protocol
6
+ module Protocol
7
+ def self.parse_request(payload)
8
+ payload = sanitised_payload(payload)
9
+ data = sender = nil
10
+ if payload.kind_of? Hash
11
+ data = payload['data']
12
+ sender = payload['sender']
13
+ else
14
+ data = payload
15
+ end
16
+ [data, sender]
17
+ end
18
+
19
+ def self.parse_response(payload)
20
+ payload = sanitised_payload(payload)
21
+ data = sender = nil
22
+ if payload.kind_of? Hash
23
+ data = payload['data']
24
+ sender = payload['sender']
25
+ code = payload['code']
26
+ else
27
+ data = payload
28
+ end
29
+ [data, code, sender]
30
+ end
31
+
32
+ def self.create_request(data, code = nil, sender = nil)
33
+ sender ||= SecureRandom.hex
34
+ payload = JSON.generate({ 'data' => data, 'code' => code,
35
+ 'sender' => sender })
36
+ [payload, sender]
37
+ end
38
+
39
+ def self.sanitised_payload(raw)
40
+ ret = begin
41
+ JSON.parse(raw)
42
+ rescue
43
+ raw
44
+ end
45
+ ret
46
+ end
47
+ end
48
+ end
@@ -0,0 +1,224 @@
1
+ # encoding: utf-8
2
+
3
+ require "logger"
4
+
5
+ # Top level module
6
+ module Gilmour
7
+ # The Responder module that provides the request and respond
8
+ # DSL
9
+ # The public methods in this class are available to be called
10
+ # from the body of the handlers directly
11
+
12
+ class Responder
13
+ attr_reader :logger
14
+ attr_reader :request
15
+
16
+ def make_logger
17
+ logger = Logger.new(STDERR)
18
+ original_formatter = Logger::Formatter.new
19
+ loglevel = ENV["LOG_LEVEL"] ? ENV["LOG_LEVEL"].to_sym : :warn
20
+ logger.level = Gilmour::LoggerLevels[loglevel] || Logger::WARN
21
+ logger.formatter = proc do |severity, datetime, progname, msg|
22
+ original_formatter.call(severity, datetime, @sender, msg)
23
+ end
24
+ logger
25
+ end
26
+
27
+ def initialize(sender, topic, data, backend, timeout=600, forked=false)
28
+ @sender = sender
29
+ @request = Mash.new(topic: topic, body: data)
30
+ @response = { data: nil, code: nil }
31
+ @backend = backend
32
+ @timeout = timeout || 600
33
+ @multi_process = forked || false
34
+ @pipe = IO.pipe
35
+ @publish_pipe = IO.pipe
36
+ @logger = make_logger()
37
+ end
38
+
39
+ def receive_data(data)
40
+ sender, res_data, res_code = JSON.parse(data)
41
+ write_response(sender, res_data, res_code) if sender && res_code
42
+ end
43
+
44
+ # Called by parent
45
+ def write_response(sender, data, code)
46
+ if code >= 300 && @backend.report_errors?
47
+ emit_error data, code
48
+ end
49
+
50
+ @backend.send_response(sender, data, code)
51
+ end
52
+
53
+ # Adds a dynamic listener for _topic_
54
+ def add_listener(topic, &handler)
55
+ if @multi_process
56
+ GLogger.error "Dynamic listeners using add_listener not supported \
57
+ in forked responder. Ignoring!"
58
+ end
59
+
60
+ @backend.add_listener(topic, &handler)
61
+ end
62
+
63
+ # Sends a response with _body_ and _code_
64
+ # If +opts[:now]+ is true, the response is sent immediately,
65
+ # else it is defered until the handler finishes executing
66
+ def respond(body, code = 200, opts = {})
67
+ @response[:data] = body
68
+ @response[:code] = code
69
+ if opts[:now]
70
+ send_response
71
+ @response = {}
72
+ end
73
+ end
74
+
75
+ # Called by parent
76
+ # :nodoc:
77
+ def execute(handler)
78
+ if @multi_process
79
+ GLogger.debug "Executing #{@sender} in forked moode"
80
+ @read_pipe = @pipe[0]
81
+ @write_pipe = @pipe[1]
82
+
83
+ @read_publish_pipe = @publish_pipe[0]
84
+ @write_publish_pipe = @publish_pipe[1]
85
+
86
+ pid = Process.fork do
87
+ @backend.stop
88
+ EventMachine.stop_event_loop
89
+ @read_pipe.close
90
+ @read_publish_pipe.close
91
+ @response_sent = false
92
+ _execute(handler)
93
+ end
94
+
95
+ @write_pipe.close
96
+ @write_publish_pipe.close
97
+
98
+ pub_mutex = Mutex.new
99
+
100
+ pub_reader = Thread.new {
101
+ loop {
102
+ begin
103
+ data = @read_publish_pipe.readline
104
+ pub_mutex.synchronize do
105
+ destination, message = JSON.parse(data)
106
+ @backend.publish(message, destination)
107
+ end
108
+ rescue EOFError
109
+ # awkward blank rescue block
110
+ rescue Exception => e
111
+ GLogger.debug e.message
112
+ GLogger.debug e.backtrace
113
+ end
114
+ }
115
+ }
116
+
117
+ begin
118
+ receive_data(@read_pipe.readline)
119
+ rescue EOFError => e
120
+ logger.debug e.message
121
+ logger.debug "EOFError caught in responder.rb, because of nil response"
122
+ end
123
+
124
+ pid, status = Process.waitpid2(pid)
125
+ if !status
126
+ msg = "Child Process #{pid} crashed without status."
127
+ logger.error msg
128
+ # Set the multi-process mode as false, the child has died anyway.
129
+ @multi_process = false
130
+ write_response(@sender, msg, 500)
131
+ elsif status.exitstatus > 0
132
+ msg = "Child Process #{pid} exited with status #{status.exitstatus}"
133
+ logger.error msg
134
+ # Set the multi-process mode as false, the child has died anyway.
135
+ @multi_process = false
136
+ write_response(@sender, msg, 500)
137
+ end
138
+
139
+ pub_mutex.synchronize do
140
+ pub_reader.kill
141
+ end
142
+
143
+ @read_pipe.close
144
+ @read_publish_pipe.close
145
+ else
146
+ _execute(handler)
147
+ end
148
+ end
149
+
150
+ def emit_error(message, code=500, extra={})
151
+ # Publish all errors on gilmour.error
152
+ # This may or may not have a listener based on the configuration
153
+ # supplied at setup.
154
+ opts = {
155
+ :topic => @request[:topic],
156
+ :data => @request[:data],
157
+ :description => '',
158
+ :sender => @sender,
159
+ :multi_process => @multi_process,
160
+ :code => 500
161
+ }.merge(extra || {})
162
+
163
+ opts[:timestamp] = Time.now.getutc
164
+ payload = {:traceback => message, :extra => opts, :code => code}
165
+ @backend.emit_error payload
166
+ end
167
+
168
+ # Called by child
169
+ # :nodoc:
170
+ def _execute(handler)
171
+ begin
172
+ Timeout.timeout(@timeout) do
173
+ instance_eval(&handler)
174
+ end
175
+ rescue Timeout::Error => e
176
+ logger.error e.message
177
+ logger.error e.backtrace
178
+ @response[:code] = 504
179
+ @response[:data] = e.message
180
+ rescue Exception => e
181
+ logger.error e.message
182
+ logger.error e.backtrace
183
+ @response[:code] = 500
184
+ @response[:data] = e.message
185
+ end
186
+
187
+ send_response if @response[:code]
188
+ end
189
+
190
+ # Publishes a message. See Backend::publish
191
+ def publish(message, destination, opts = {}, code=nil)
192
+ if @multi_process
193
+ if block_given?
194
+ GLogger.error "Publish callback not supported in forked responder. Ignoring!"
195
+ # raise Exception.new("Publish Callback is not supported in forked mode.")
196
+ end
197
+
198
+ msg = JSON.generate([destination, message, code])
199
+ @write_publish_pipe.write(msg+"\n")
200
+ @write_publish_pipe.flush
201
+ elsif block_given?
202
+ blk = Proc.new
203
+ @backend.publish(message, destination, opts, &blk)
204
+ else
205
+ @backend.publish(message, destination, opts)
206
+ end
207
+ end
208
+
209
+ # Called by child
210
+ # :nodoc:
211
+ def send_response
212
+ return if @response_sent
213
+ @response_sent = true
214
+
215
+ if @multi_process
216
+ msg = JSON.generate([@sender, @response[:data], @response[:code]])
217
+ @write_pipe.write(msg+"\n")
218
+ @write_pipe.flush # This flush is very important
219
+ else
220
+ write_response(@sender, @response[:data], @response[:code])
221
+ end
222
+ end
223
+ end
224
+ end
@@ -0,0 +1,20 @@
1
+ module Gilmour
2
+ class Waiter
3
+ def initialize
4
+ @waiter_m = Mutex.new
5
+ @waiter_c = ConditionVariable.new
6
+ end
7
+
8
+ def synchronize(&blk)
9
+ @waiter_m.synchronize(&blk)
10
+ end
11
+
12
+ def signal
13
+ synchronize { @waiter_c.signal }
14
+ end
15
+
16
+ def wait(timeout=nil)
17
+ synchronize { @waiter_c.wait(@waiter_m, timeout) }
18
+ end
19
+ end
20
+ end
data/lib/gilmour.rb ADDED
@@ -0,0 +1,11 @@
1
+ # encoding: utf-8
2
+
3
+ # This is required to check whether Mash class already exists
4
+ def class_exists?(class_name)
5
+ klass = Module.const_get(class_name)
6
+ return klass.is_a?(Class)
7
+ rescue NameError
8
+ return false
9
+ end
10
+
11
+ require_relative 'gilmour/base'
@@ -0,0 +1,25 @@
1
+
2
+ class Waiter
3
+ def initialize
4
+ @waiter_m = Mutex.new
5
+ @waiter_c = ConditionVariable.new
6
+ end
7
+
8
+ def synchronize(&blk)
9
+ @waiter_m.synchronize(&blk)
10
+ end
11
+
12
+ def signal
13
+ synchronize { @waiter_c.signal }
14
+ end
15
+
16
+ def wait(timeout=nil)
17
+ synchronize { @waiter_c.wait(@waiter_m, timeout) }
18
+ end
19
+ end
20
+
21
+ RSpec.configure do |config|
22
+ config.expect_with :rspec do |c|
23
+ c.syntax = [:should, :expect]
24
+ end
25
+ end
@@ -0,0 +1,58 @@
1
+ require 'yaml'
2
+ require 'json'
3
+ require 'securerandom'
4
+ require 'fiber'
5
+ require '../lib/gilmour/protocol'
6
+ require 'em-hiredis'
7
+
8
+ require_relative 'common'
9
+
10
+ def options(which)
11
+ data = YAML::load(File.open("#{File.dirname(__FILE__)}/data.yml"))
12
+ data[which]
13
+ end
14
+
15
+ def redis_connection_options
16
+ options(:connection)
17
+ end
18
+
19
+ def redis_ping_options
20
+ options(:ping)
21
+ end
22
+
23
+ def redis_wildcard_options
24
+ options(:wildcard)
25
+ end
26
+
27
+ def redis_publish_async(options, message, key)
28
+ operation = proc do
29
+ redis = EM::Hiredis.connect
30
+ payload, _ = Gilmour::Protocol.create_request(message)
31
+ redis.publish(key, payload)
32
+ end
33
+ EM.defer(operation)
34
+ end
35
+
36
+ def redis_send_and_recv(options, message, key)
37
+ waiter = Waiter.new
38
+ response = code = nil
39
+ operation = proc do
40
+ redis = EM::Hiredis.connect
41
+ payload, sender = Gilmour::Protocol.create_request(message)
42
+ response_topic = "gilmour.response.#{sender}"
43
+ redis.pubsub.subscribe(response_topic)
44
+ redis.pubsub.on(:message) do |topic, data|
45
+ begin
46
+ response, code, _ = Gilmour::Protocol.parse_response(data)
47
+ waiter.signal
48
+ rescue Exception => e
49
+ $stderr.puts e.message
50
+ end
51
+ end
52
+ redis.publish(key, payload)
53
+ end
54
+ EM.defer(operation)
55
+ waiter.wait
56
+ [response, code]
57
+ end
58
+
@@ -0,0 +1,12 @@
1
+ ---
2
+ :connection:
3
+ :host: localhost
4
+ :exchange: testtopicexchange
5
+ :ping:
6
+ :message: Ping!
7
+ :response: Pong!
8
+ :wildcard:
9
+ :message: Wild!
10
+ :topic: test.wildcard.foo
11
+
12
+
@@ -0,0 +1,48 @@
1
+ # encoding: utf-8
2
+
3
+ require 'rspec/given'
4
+ require 'amqp'
5
+ require 'redis'
6
+
7
+ require_relative 'helpers/connection'
8
+ require './testservice/test_service_base'
9
+
10
+ describe TestServiceBase do
11
+ Given(:subscriber) { TestServiceBase }
12
+ Then { subscriber.should respond_to(:subscribers) }
13
+ Then { subscriber.subscribers.should be_kind_of(Hash) }
14
+
15
+ context 'Load existing subscribers' do
16
+ modules_dir = './testservice/subscribers'
17
+ modules = Dir["#{modules_dir}/*.rb"]
18
+ When { subscriber.load_all(modules_dir) }
19
+ Then do
20
+ subscribers = subscriber.subscribers.map do |topic, handlers|
21
+ handlers.map { |handler| handler[:subscriber] }
22
+ end.flatten.uniq
23
+ subscribers.size.should == modules.size
24
+ end
25
+ end
26
+ # context 'Connect to AMQP' do
27
+ # after(:all) do
28
+ # AMQP.stop
29
+ # EM.stop
30
+ # end
31
+ # Given(:subscriber) { TestServiceBase.new(amqp_connection_options, 'amqp') }
32
+ # Given(:backend) { subscriber.backends['amqp'] }
33
+ # Then { backend.connection.should be_kind_of AMQP::Session }
34
+ # And { backend.connection.connected?.should be_true }
35
+ # And { backend.channel.should be_kind_of AMQP::Channel }
36
+ # And { backend.exchange.should be_kind_of AMQP::Exchange }
37
+ # And { backend.exchange.type.should == :topic }
38
+ # end
39
+
40
+ context 'Connect to Redis' do
41
+ Given(:subscriber) do
42
+ TestServiceBase.new(redis_connection_options, 'redis')
43
+ end
44
+ Given(:backend) { subscriber.backends['redis'] }
45
+ Then { backend.subscriber.should be_kind_of EM::Hiredis::PubsubClient }
46
+ And { backend.publisher.should be_kind_of EM::Hiredis::Client }
47
+ end
48
+ end