banter 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,52 @@
1
+
2
+ # Internal database logger that saves the publish message before it is send
3
+ # and also acts as a sink for messages.
4
+ module Banter
5
+ class DbLogger
6
+
7
+ def initialize
8
+ @config = Configuration.configuration[:mongo]
9
+ if @config.present?
10
+ @client = Mongo::MongoClient.new(@config[:host], @config[:port])
11
+ else
12
+ @client = Mongo::MongoClient.new
13
+ end
14
+
15
+ @db = @client.db("logger")
16
+
17
+ end
18
+
19
+ def start_subscribe
20
+ subscriber = ::Banter::Server::RabbitMQSubscriber.new("")
21
+ subscriber.start("logger") do |info, properties, contents|
22
+ # Write to mongo the contents and the routing_key, with the routing_key being indexed
23
+ routing_key = info[:routing_key]
24
+ chunks = routing_key.split(".")
25
+ # Logger will split the data into collections based upon the first parameter of the routing_key (initial roll out will only use 'honest')
26
+ collection_name = chunks[1] || chunks[0]
27
+ collection = @db.collection(collection_name)
28
+ # no need for safe writes. need to write to a file log as well.
29
+ collection.insert({routing_key: routing_key, cts: contents, ts: properties[:timestamp]})
30
+ # make sure that we always index these two items in mongo
31
+ # it's a small price of a call, so no big deal
32
+ collection.ensure_index( {routing_key:1, ts:1} )
33
+ end
34
+ end
35
+
36
+ def log_publish(routing_key, message)
37
+ collection_name = "publisher"
38
+ collection = @db.collection(collection_name)
39
+
40
+ # no need for safe writes. need to write to a file log as well.
41
+ collection.insert({routing_key: routing_key, cts: message[:payload], ts: message[:ts], pub: message[:pub]})
42
+ # make sure that we always index these two items in mongo
43
+ # it's a small price of a call, so no big deal
44
+ collection.ensure_index( {routing_key:1, ts:1} )
45
+
46
+ end
47
+
48
+ def teardown
49
+ @subscriber.teardown
50
+ end
51
+ end
52
+ end
@@ -0,0 +1,4 @@
1
+ module Banter
2
+ class PayloadValidationError < StandardError
3
+ end
4
+ end
@@ -0,0 +1,49 @@
1
+ # Internal log class that is used to log messages before sending, after receiving, and failure to send
2
+ module Banter
3
+ class Logger
4
+ def initialize(enable_publish_logging = true)
5
+ @enabled = Banter::Configuration.configuration[:logger][:enabled]
6
+ @enabled = enable_publish_logging if @enabled.nil?
7
+ end
8
+
9
+ def log_publish(routing_key, message)
10
+ return unless @enabled
11
+ # generate tags needed
12
+ tags = ["PUBLISH", message[:ts], message[:pub], message[:v], routing_key]
13
+
14
+ # FIX!!! -thl
15
+ # Could logging like this be too slow?
16
+ # Or should it be threaded?
17
+ # We'll need to benchmark, as we don't want this to get too slow.
18
+ logger.tagged(tags) { logger.warn message[:payload].as_json }
19
+
20
+ # TODO: -thl
21
+ # Log it into mongo as well?
22
+ end
23
+
24
+ def failed_publish(routing_key, properties, message)
25
+ tags = ["FAILED_SEND", message[:ts], message[:pub], message[:v], routing_key]
26
+ logger.tagged(tags) { logger.warn message[:payload].as_json }
27
+ end
28
+
29
+ def log_receive(routing_key, message)
30
+ tags = ["RECEIVED", message[:ts], message[:pub], message[:v], routing_key, Process::pid]
31
+ logger.tagged(tags) { logger.warn message[:payload].as_json }
32
+ end
33
+
34
+ def log_service(service_name, log_level, message)
35
+ tags = ["SERVICE", service_name, Process::pid]
36
+ log_method = log_level.to_sym.to_proc
37
+ logger.tagged(tags) { log_method.call(logger) { message.as_json } }
38
+ end
39
+
40
+ def teardown
41
+ logger.close
42
+ end
43
+
44
+ private
45
+ def logger
46
+ @logger ||= ActiveSupport::TaggedLogging.new(Banter.logger)
47
+ end
48
+ end
49
+ end
@@ -0,0 +1,42 @@
1
+ require 'logger'
2
+
3
+ # Manages a logger that can be used throughout the pubsub gem
4
+ # The logger can be set to any object that quacks like a logger including the
5
+ # Rails logger if so desired
6
+ #
7
+ # The logger can be set in the initializer of a program or anywhere throughout as
8
+ # Banter.logger = Rails.logger for example
9
+
10
+ module Banter
11
+ module Logging
12
+ # Sets the logger
13
+ #
14
+ # @param logger The logger to use throughout the pubsub gem
15
+ # @return The logger we use throughout the gem
16
+ def self.logger=(logger)
17
+ raise StandardError("Can't set logger to nil") unless logger.present?
18
+ @logger = logger
19
+ end
20
+
21
+ # Gets the logger
22
+ # If no logger is defined when this method is called it will return a standard
23
+ # ruby logger
24
+ #
25
+ # @return The logger we use throughout the gem
26
+ def self.logger
27
+ config = Banter::Configuration.configuration[:logger]
28
+ @logger ||= create_logger( {}.
29
+ merge( config[:level].present? ? { log_level: ::Logger::Severity.const_get(config[:level].upcase) } : {} ).
30
+ merge( config[:file].present? ? { log_target: config[:file] } : {} ) )
31
+ end
32
+
33
+ # Builds a standard ruby Logger
34
+ #
35
+ # @return Returns the logger at the end of the method
36
+ def self.create_logger( options = {} )
37
+ @logger = ::Logger.new(options.fetch(:log_target){ STDOUT })
38
+ @logger.level = options.fetch(:log_level){ ::Logger::INFO }
39
+ @logger
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,50 @@
1
+
2
+ require "hashie"
3
+ require "json"
4
+
5
+ module Banter
6
+ class Message
7
+ @@message_version = "1"
8
+ attr_reader :ts
9
+ attr_reader :pub
10
+ attr_reader :version
11
+ attr_reader :payload
12
+ attr_reader :context
13
+
14
+ def initialize
15
+ end
16
+
17
+ # Context object should be passed into the call.
18
+ def serialize(context, routing_key, payload)
19
+ @ts = Time.now.to_i
20
+ @pub = "#{routing_key}:#{Socket.gethostname()}:#{Process::pid}"
21
+ @version = @@message_version
22
+ @payload = payload
23
+ @context = context
24
+ to_hash
25
+ end
26
+
27
+ def parse(envelope)
28
+ contents = Hashie::Mash.new(::JSON.parse(envelope))
29
+ @ts = contents[:ts]
30
+ @pub = contents[:pub]
31
+ # Version used for messaging can get updated if we need to add extra header information, and need
32
+ # to move each section of the library separately.
33
+ @version = contents[:v]
34
+ @payload = contents[:payload]
35
+ @context = ::Banter::Context.from_json(contents[:context])
36
+ to_hash
37
+ end
38
+
39
+ def to_hash
40
+ ::Hashie::Mash.new({
41
+ ts: @ts,
42
+ pub: @pub,
43
+ v: @@message_version,
44
+ payload: @payload,
45
+ context: @context.as_json
46
+ })
47
+ end
48
+
49
+ end
50
+ end
@@ -0,0 +1,14 @@
1
+ # Middleware to ensure that we clear the context with every request
2
+
3
+ module Banter
4
+ class Middleware
5
+ def initialize(app)
6
+ @app = app
7
+ end
8
+
9
+ def call(env)
10
+ Banter::Context.clear!
11
+ @app.call(env)
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,93 @@
1
+ module Banter
2
+ class Publisher
3
+ attr_reader :publisher
4
+ attr_reader :channel
5
+
6
+ @@publisher = nil
7
+
8
+ def self.instance
9
+ if @@publisher.nil?
10
+ @@publisher = ::Banter::Publisher.new
11
+ end
12
+ @@publisher
13
+ end
14
+
15
+ def initialize(exchange="honest")
16
+ @exchange = exchange
17
+ @disabled = false
18
+ @logger = ::Banter::Logger.new()
19
+
20
+ @config = Configuration.configuration
21
+
22
+ self
23
+ end
24
+
25
+ def enable(value)
26
+ @disabled = !value
27
+ @disabled
28
+ end
29
+
30
+ def start
31
+ if !@config[:enabled].nil? && @config[:enabled] == false
32
+ @disabled = true
33
+ return
34
+ end
35
+
36
+ # grab server configuration from initialization file somewhere
37
+ begin
38
+ @connection = Bunny.new(Configuration.configuration[:connection])
39
+ @connection.start
40
+
41
+ @channel = @connection.create_channel
42
+ @publisher = @channel.topic(@exchange, :durable=>true, :auto_delete=>false)
43
+
44
+ rescue => e
45
+ Airbrake.notify(e, parameters: {message: e.message}, environment_name: ENV['RAILS_ENV'] )
46
+ return
47
+ end
48
+
49
+
50
+ @publisher.on_return do |return_info, properties, content|
51
+ # contents are already transformed into message that we want to send
52
+ @logger.failed_publish(return_info[:routing_key], properties, content)
53
+ end
54
+
55
+ end
56
+
57
+ def publish(context, key, payload, enabled = true)
58
+ routing_key = "#{@exchange}.#{key}"
59
+ envelope = ::Banter::Message.new.serialize(context, key, payload)
60
+
61
+ if @publisher.nil?
62
+ start
63
+ end
64
+
65
+ if @disabled || @publisher.nil? || !enabled
66
+ @logger.failed_publish(routing_key, {}, envelope)
67
+ else
68
+ tries = 2
69
+ begin
70
+ @publisher.publish(envelope.to_json, :persistent=>true, :mandatory=>true, :timestamp=>envelope[:ts], :content_type=>"application/json", :routing_key =>routing_key )
71
+ @logger.log_publish(routing_key, envelope)
72
+ rescue => e
73
+ tries -= 1
74
+ teardown
75
+ start
76
+ if tries > 0 && @publisher
77
+ retry
78
+ else
79
+ @logger.failed_publish(routing_key, {}, envelope)
80
+ end
81
+ end
82
+
83
+ end
84
+
85
+ end
86
+
87
+ def teardown
88
+ @connection.close
89
+ @publisher = nil
90
+ end
91
+
92
+ end
93
+ end
@@ -0,0 +1,27 @@
1
+ # This railtie provides Rails engine hooks
2
+
3
+ module Banter
4
+ class Railtie < ::Rails::Railtie
5
+ initializer "banter.insert_middleware" do |app|
6
+ app.config.middleware.use Banter::Middleware
7
+ end
8
+
9
+ console do
10
+ # Set up basic context to identify messages getting generated from console
11
+ Banter::Context.setup_context(
12
+ application: "#{Banter::Configuration.application_name}/console",
13
+ hostname: `hostname`.strip,
14
+ unique_id: Digest::SHA1.new.to_s
15
+ )
16
+ end
17
+
18
+ rake_tasks do
19
+ # Set up basic context to identify messages getting generated from rake tasks
20
+ Banter::Context.setup_context(
21
+ application: "#{Banter::Configuration.application_name}/rake",
22
+ hostname: `hostname`.strip,
23
+ unique_id: Digest::SHA1.new.to_s
24
+ )
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,7 @@
1
+ module Banter
2
+ module Server
3
+ end
4
+ end
5
+
6
+ require 'banter/server/client_queue_listener'
7
+ require 'banter/server/subscriber_server'
@@ -0,0 +1,56 @@
1
+ require 'rubygems'
2
+ require 'celluloid/autostart'
3
+ require 'celluloid/io'
4
+ require 'banter'
5
+
6
+ module Banter
7
+ module Server
8
+ class ClientQueueListener
9
+ include Celluloid
10
+
11
+ attr_reader :worker_class
12
+
13
+ def initialize(worker_class, request_key, queue, durable = true, topic = "honest")
14
+ @topic = topic
15
+ @request_key = request_key
16
+ @worker_class = worker_class
17
+ @queue_name = queue
18
+ @durable = durable
19
+ @subscriber = ::Banter::Server::RabbitMQSubscriber.new(@request_key, @durable, @topic)
20
+ end
21
+
22
+ def start
23
+ @subscriber.start(@queue_name, false) do |delivery_info, properties, envelope|
24
+ message_received(delivery_info, properties, envelope)
25
+ true
26
+ end
27
+ end
28
+
29
+ def shutdown
30
+ @subscriber.teardown if @subscriber.present?
31
+ # TODO -thl
32
+ # This the kosher thing to do in ruby?
33
+ teardown
34
+ end
35
+
36
+ def message_received(delivery_info, properties, envelope)
37
+ Banter.logger.debug("Message received by listener on #{worker_class.name} with payload: #{envelope[:payload]}")
38
+ worker = worker_class.new(delivery_info, properties, envelope[:context])
39
+ worker.perform!(envelope[:payload])
40
+ rescue => e
41
+ Banter.logger.error("Failed message for #{worker_class.name}")
42
+ Airbrake.notify("Failed perform for #{worker_class.name}",
43
+ params: {
44
+ info: delivery_info,
45
+ properties: properties,
46
+ context: envelope[:context],
47
+ payload: envelope[:payload] })
48
+
49
+ end
50
+
51
+ protected
52
+ def teardown
53
+ end
54
+ end
55
+ end
56
+ end
@@ -0,0 +1,75 @@
1
+ module Banter
2
+ module Server
3
+
4
+ class RabbitMQSubscriber
5
+ attr_reader :listener
6
+ attr_reader :exchange
7
+ attr_reader :channel
8
+
9
+ def initialize(routing_key, durable = true, topic="honest")
10
+ @initial_key = routing_key
11
+ @durable = durable
12
+ @topic = topic
13
+
14
+ if @initial_key.present?
15
+ @routing_key = "#{@topic}.#{@initial_key}.#"
16
+ else
17
+ @routing_key = "#{@topic}.#"
18
+ end
19
+ @logger = ::Banter::Logger.new
20
+
21
+ self
22
+ end
23
+
24
+ # name - used to ensure that certain consumers are actually listening to an exchange
25
+ # pass in a lambda for this method to work. We might only want to expose the content instead of
26
+ # all 3 chunks.
27
+ def start(name, blocking=false)
28
+ @connection = Bunny.new(Configuration.configuration[:connection])
29
+ begin
30
+ @connection.start
31
+ rescue => e
32
+ Airbrake.notify("RabbitMQ unreachable!", params: { message: e.message }, environment_name: ENV['RAILS_ENV'])
33
+ raise e
34
+ end
35
+
36
+ @channel = @connection.create_channel
37
+ @exchange = @channel.topic(@topic, :durable => @durable, :auto_delete => false)
38
+
39
+ # FIX!!! -thl
40
+ # Need to ensure that the ids for a server will be reproducible in case a server
41
+ # goes down and has to get restarted.
42
+ if @initial_key.present?
43
+ @queue = "#{@initial_key}.#{name}"
44
+ else
45
+ @queue = "#{name}"
46
+ end
47
+
48
+ queue_arguments = {}
49
+ queue_arguments["x-dead-letter-exchange"] = Configuration.configuration[:dead_letter] if Configuration.configuration[:dead_letter].present?
50
+ queue_arguments["x-message-ttl"] = Configuration.default_queue_ttl
51
+ @listener = @channel.queue(@queue, arguments: queue_arguments).bind(@exchange, routing_key: @routing_key, exclusive: false)
52
+
53
+ # Parameters for subscribe that might be useful:
54
+ # :block=>true - Used for long running consumer applications. (backend servers?)
55
+ @consumer = @listener.subscribe(consumer_tag: name, block: blocking)
56
+
57
+ @consumer.on_delivery do |delivery_info, properties, contents|
58
+ Banter.logger.debug("Message delivery with contents: #{contents}")
59
+ if delivery_info[:redelivered]
60
+ Airbrake.notify("PubSub Message redelivery", params: { info: delivery_info, props: properties, contents: contents }, environment_name: ENV['RAILS_ENV'])
61
+ end
62
+ message = ::Banter::Message.new.parse(contents)
63
+ @logger.log_receive(delivery_info[:routing_key], message)
64
+ yield delivery_info, properties, message
65
+ true
66
+ end
67
+ end
68
+
69
+ def teardown
70
+ @consumer.cancel if @consumer.present?
71
+ @connection.close if @connection.present?
72
+ end
73
+ end
74
+ end
75
+ end