banter 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +22 -0
- data/.ruby-version +1 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +113 -0
- data/Rakefile +1 -0
- data/banter.gemspec +37 -0
- data/bin/start_subscribers +14 -0
- data/config/pubsub.yml +17 -0
- data/lib/banter.rb +47 -0
- data/lib/banter/cli.rb +94 -0
- data/lib/banter/configuration.rb +42 -0
- data/lib/banter/context.rb +70 -0
- data/lib/banter/db_logger.rb +52 -0
- data/lib/banter/exceptions/payload_validation_error.rb +4 -0
- data/lib/banter/logger.rb +49 -0
- data/lib/banter/logging.rb +42 -0
- data/lib/banter/message.rb +50 -0
- data/lib/banter/middleware.rb +14 -0
- data/lib/banter/publisher.rb +93 -0
- data/lib/banter/railtie.rb +27 -0
- data/lib/banter/server.rb +7 -0
- data/lib/banter/server/client_queue_listener.rb +56 -0
- data/lib/banter/server/rabbit_mq_subscriber.rb +75 -0
- data/lib/banter/server/subscriber_server.rb +84 -0
- data/lib/banter/subscriber.rb +100 -0
- data/lib/banter/version.rb +3 -0
- data/spec/banter/cli_spec.rb +145 -0
- data/spec/banter/server/client_queue_listener_spec.rb +76 -0
- data/spec/banter/server/client_worker_spec.rb +161 -0
- data/spec/banter/server/rabbitmq_subscriber_spec.rb +5 -0
- data/spec/config.yml +20 -0
- data/spec/logger_spec.rb +110 -0
- data/spec/message_spec.rb +65 -0
- data/spec/spec_helper.rb +49 -0
- metadata +261 -0
@@ -0,0 +1,52 @@
|
|
1
|
+
|
2
|
+
# Internal database logger that saves the publish message before it is send
|
3
|
+
# and also acts as a sink for messages.
|
4
|
+
module Banter
|
5
|
+
class DbLogger
|
6
|
+
|
7
|
+
def initialize
|
8
|
+
@config = Configuration.configuration[:mongo]
|
9
|
+
if @config.present?
|
10
|
+
@client = Mongo::MongoClient.new(@config[:host], @config[:port])
|
11
|
+
else
|
12
|
+
@client = Mongo::MongoClient.new
|
13
|
+
end
|
14
|
+
|
15
|
+
@db = @client.db("logger")
|
16
|
+
|
17
|
+
end
|
18
|
+
|
19
|
+
def start_subscribe
|
20
|
+
subscriber = ::Banter::Server::RabbitMQSubscriber.new("")
|
21
|
+
subscriber.start("logger") do |info, properties, contents|
|
22
|
+
# Write to mongo the contents and the routing_key, with the routing_key being indexed
|
23
|
+
routing_key = info[:routing_key]
|
24
|
+
chunks = routing_key.split(".")
|
25
|
+
# Logger will split the data into collections based upon the first parameter of the routing_key (initial roll out will only use 'honest')
|
26
|
+
collection_name = chunks[1] || chunks[0]
|
27
|
+
collection = @db.collection(collection_name)
|
28
|
+
# no need for safe writes. need to write to a file log as well.
|
29
|
+
collection.insert({routing_key: routing_key, cts: contents, ts: properties[:timestamp]})
|
30
|
+
# make sure that we always index these two items in mongo
|
31
|
+
# it's a small price of a call, so no big deal
|
32
|
+
collection.ensure_index( {routing_key:1, ts:1} )
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def log_publish(routing_key, message)
|
37
|
+
collection_name = "publisher"
|
38
|
+
collection = @db.collection(collection_name)
|
39
|
+
|
40
|
+
# no need for safe writes. need to write to a file log as well.
|
41
|
+
collection.insert({routing_key: routing_key, cts: message[:payload], ts: message[:ts], pub: message[:pub]})
|
42
|
+
# make sure that we always index these two items in mongo
|
43
|
+
# it's a small price of a call, so no big deal
|
44
|
+
collection.ensure_index( {routing_key:1, ts:1} )
|
45
|
+
|
46
|
+
end
|
47
|
+
|
48
|
+
def teardown
|
49
|
+
@subscriber.teardown
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
# Internal log class that is used to log messages before sending, after receiving, and failure to send
|
2
|
+
module Banter
|
3
|
+
class Logger
|
4
|
+
def initialize(enable_publish_logging = true)
|
5
|
+
@enabled = Banter::Configuration.configuration[:logger][:enabled]
|
6
|
+
@enabled = enable_publish_logging if @enabled.nil?
|
7
|
+
end
|
8
|
+
|
9
|
+
def log_publish(routing_key, message)
|
10
|
+
return unless @enabled
|
11
|
+
# generate tags needed
|
12
|
+
tags = ["PUBLISH", message[:ts], message[:pub], message[:v], routing_key]
|
13
|
+
|
14
|
+
# FIX!!! -thl
|
15
|
+
# Could logging like this be too slow?
|
16
|
+
# Or should it be threaded?
|
17
|
+
# We'll need to benchmark, as we don't want this to get too slow.
|
18
|
+
logger.tagged(tags) { logger.warn message[:payload].as_json }
|
19
|
+
|
20
|
+
# TODO: -thl
|
21
|
+
# Log it into mongo as well?
|
22
|
+
end
|
23
|
+
|
24
|
+
def failed_publish(routing_key, properties, message)
|
25
|
+
tags = ["FAILED_SEND", message[:ts], message[:pub], message[:v], routing_key]
|
26
|
+
logger.tagged(tags) { logger.warn message[:payload].as_json }
|
27
|
+
end
|
28
|
+
|
29
|
+
def log_receive(routing_key, message)
|
30
|
+
tags = ["RECEIVED", message[:ts], message[:pub], message[:v], routing_key, Process::pid]
|
31
|
+
logger.tagged(tags) { logger.warn message[:payload].as_json }
|
32
|
+
end
|
33
|
+
|
34
|
+
def log_service(service_name, log_level, message)
|
35
|
+
tags = ["SERVICE", service_name, Process::pid]
|
36
|
+
log_method = log_level.to_sym.to_proc
|
37
|
+
logger.tagged(tags) { log_method.call(logger) { message.as_json } }
|
38
|
+
end
|
39
|
+
|
40
|
+
def teardown
|
41
|
+
logger.close
|
42
|
+
end
|
43
|
+
|
44
|
+
private
|
45
|
+
def logger
|
46
|
+
@logger ||= ActiveSupport::TaggedLogging.new(Banter.logger)
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
require 'logger'
|
2
|
+
|
3
|
+
# Manages a logger that can be used throughout the pubsub gem
|
4
|
+
# The logger can be set to any object that quacks like a logger including the
|
5
|
+
# Rails logger if so desired
|
6
|
+
#
|
7
|
+
# The logger can be set in the initializer of a program or anywhere throughout as
|
8
|
+
# Banter.logger = Rails.logger for example
|
9
|
+
|
10
|
+
module Banter
|
11
|
+
module Logging
|
12
|
+
# Sets the logger
|
13
|
+
#
|
14
|
+
# @param logger The logger to use throughout the pubsub gem
|
15
|
+
# @return The logger we use throughout the gem
|
16
|
+
def self.logger=(logger)
|
17
|
+
raise StandardError("Can't set logger to nil") unless logger.present?
|
18
|
+
@logger = logger
|
19
|
+
end
|
20
|
+
|
21
|
+
# Gets the logger
|
22
|
+
# If no logger is defined when this method is called it will return a standard
|
23
|
+
# ruby logger
|
24
|
+
#
|
25
|
+
# @return The logger we use throughout the gem
|
26
|
+
def self.logger
|
27
|
+
config = Banter::Configuration.configuration[:logger]
|
28
|
+
@logger ||= create_logger( {}.
|
29
|
+
merge( config[:level].present? ? { log_level: ::Logger::Severity.const_get(config[:level].upcase) } : {} ).
|
30
|
+
merge( config[:file].present? ? { log_target: config[:file] } : {} ) )
|
31
|
+
end
|
32
|
+
|
33
|
+
# Builds a standard ruby Logger
|
34
|
+
#
|
35
|
+
# @return Returns the logger at the end of the method
|
36
|
+
def self.create_logger( options = {} )
|
37
|
+
@logger = ::Logger.new(options.fetch(:log_target){ STDOUT })
|
38
|
+
@logger.level = options.fetch(:log_level){ ::Logger::INFO }
|
39
|
+
@logger
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
|
2
|
+
require "hashie"
|
3
|
+
require "json"
|
4
|
+
|
5
|
+
module Banter
|
6
|
+
class Message
|
7
|
+
@@message_version = "1"
|
8
|
+
attr_reader :ts
|
9
|
+
attr_reader :pub
|
10
|
+
attr_reader :version
|
11
|
+
attr_reader :payload
|
12
|
+
attr_reader :context
|
13
|
+
|
14
|
+
def initialize
|
15
|
+
end
|
16
|
+
|
17
|
+
# Context object should be passed into the call.
|
18
|
+
def serialize(context, routing_key, payload)
|
19
|
+
@ts = Time.now.to_i
|
20
|
+
@pub = "#{routing_key}:#{Socket.gethostname()}:#{Process::pid}"
|
21
|
+
@version = @@message_version
|
22
|
+
@payload = payload
|
23
|
+
@context = context
|
24
|
+
to_hash
|
25
|
+
end
|
26
|
+
|
27
|
+
def parse(envelope)
|
28
|
+
contents = Hashie::Mash.new(::JSON.parse(envelope))
|
29
|
+
@ts = contents[:ts]
|
30
|
+
@pub = contents[:pub]
|
31
|
+
# Version used for messaging can get updated if we need to add extra header information, and need
|
32
|
+
# to move each section of the library separately.
|
33
|
+
@version = contents[:v]
|
34
|
+
@payload = contents[:payload]
|
35
|
+
@context = ::Banter::Context.from_json(contents[:context])
|
36
|
+
to_hash
|
37
|
+
end
|
38
|
+
|
39
|
+
def to_hash
|
40
|
+
::Hashie::Mash.new({
|
41
|
+
ts: @ts,
|
42
|
+
pub: @pub,
|
43
|
+
v: @@message_version,
|
44
|
+
payload: @payload,
|
45
|
+
context: @context.as_json
|
46
|
+
})
|
47
|
+
end
|
48
|
+
|
49
|
+
end
|
50
|
+
end
|
@@ -0,0 +1,93 @@
|
|
1
|
+
module Banter
|
2
|
+
class Publisher
|
3
|
+
attr_reader :publisher
|
4
|
+
attr_reader :channel
|
5
|
+
|
6
|
+
@@publisher = nil
|
7
|
+
|
8
|
+
def self.instance
|
9
|
+
if @@publisher.nil?
|
10
|
+
@@publisher = ::Banter::Publisher.new
|
11
|
+
end
|
12
|
+
@@publisher
|
13
|
+
end
|
14
|
+
|
15
|
+
def initialize(exchange="honest")
|
16
|
+
@exchange = exchange
|
17
|
+
@disabled = false
|
18
|
+
@logger = ::Banter::Logger.new()
|
19
|
+
|
20
|
+
@config = Configuration.configuration
|
21
|
+
|
22
|
+
self
|
23
|
+
end
|
24
|
+
|
25
|
+
def enable(value)
|
26
|
+
@disabled = !value
|
27
|
+
@disabled
|
28
|
+
end
|
29
|
+
|
30
|
+
def start
|
31
|
+
if !@config[:enabled].nil? && @config[:enabled] == false
|
32
|
+
@disabled = true
|
33
|
+
return
|
34
|
+
end
|
35
|
+
|
36
|
+
# grab server configuration from initialization file somewhere
|
37
|
+
begin
|
38
|
+
@connection = Bunny.new(Configuration.configuration[:connection])
|
39
|
+
@connection.start
|
40
|
+
|
41
|
+
@channel = @connection.create_channel
|
42
|
+
@publisher = @channel.topic(@exchange, :durable=>true, :auto_delete=>false)
|
43
|
+
|
44
|
+
rescue => e
|
45
|
+
Airbrake.notify(e, parameters: {message: e.message}, environment_name: ENV['RAILS_ENV'] )
|
46
|
+
return
|
47
|
+
end
|
48
|
+
|
49
|
+
|
50
|
+
@publisher.on_return do |return_info, properties, content|
|
51
|
+
# contents are already transformed into message that we want to send
|
52
|
+
@logger.failed_publish(return_info[:routing_key], properties, content)
|
53
|
+
end
|
54
|
+
|
55
|
+
end
|
56
|
+
|
57
|
+
def publish(context, key, payload, enabled = true)
|
58
|
+
routing_key = "#{@exchange}.#{key}"
|
59
|
+
envelope = ::Banter::Message.new.serialize(context, key, payload)
|
60
|
+
|
61
|
+
if @publisher.nil?
|
62
|
+
start
|
63
|
+
end
|
64
|
+
|
65
|
+
if @disabled || @publisher.nil? || !enabled
|
66
|
+
@logger.failed_publish(routing_key, {}, envelope)
|
67
|
+
else
|
68
|
+
tries = 2
|
69
|
+
begin
|
70
|
+
@publisher.publish(envelope.to_json, :persistent=>true, :mandatory=>true, :timestamp=>envelope[:ts], :content_type=>"application/json", :routing_key =>routing_key )
|
71
|
+
@logger.log_publish(routing_key, envelope)
|
72
|
+
rescue => e
|
73
|
+
tries -= 1
|
74
|
+
teardown
|
75
|
+
start
|
76
|
+
if tries > 0 && @publisher
|
77
|
+
retry
|
78
|
+
else
|
79
|
+
@logger.failed_publish(routing_key, {}, envelope)
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
end
|
84
|
+
|
85
|
+
end
|
86
|
+
|
87
|
+
def teardown
|
88
|
+
@connection.close
|
89
|
+
@publisher = nil
|
90
|
+
end
|
91
|
+
|
92
|
+
end
|
93
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
# This railtie provides Rails engine hooks
|
2
|
+
|
3
|
+
module Banter
|
4
|
+
class Railtie < ::Rails::Railtie
|
5
|
+
initializer "banter.insert_middleware" do |app|
|
6
|
+
app.config.middleware.use Banter::Middleware
|
7
|
+
end
|
8
|
+
|
9
|
+
console do
|
10
|
+
# Set up basic context to identify messages getting generated from console
|
11
|
+
Banter::Context.setup_context(
|
12
|
+
application: "#{Banter::Configuration.application_name}/console",
|
13
|
+
hostname: `hostname`.strip,
|
14
|
+
unique_id: Digest::SHA1.new.to_s
|
15
|
+
)
|
16
|
+
end
|
17
|
+
|
18
|
+
rake_tasks do
|
19
|
+
# Set up basic context to identify messages getting generated from rake tasks
|
20
|
+
Banter::Context.setup_context(
|
21
|
+
application: "#{Banter::Configuration.application_name}/rake",
|
22
|
+
hostname: `hostname`.strip,
|
23
|
+
unique_id: Digest::SHA1.new.to_s
|
24
|
+
)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'celluloid/autostart'
|
3
|
+
require 'celluloid/io'
|
4
|
+
require 'banter'
|
5
|
+
|
6
|
+
module Banter
|
7
|
+
module Server
|
8
|
+
class ClientQueueListener
|
9
|
+
include Celluloid
|
10
|
+
|
11
|
+
attr_reader :worker_class
|
12
|
+
|
13
|
+
def initialize(worker_class, request_key, queue, durable = true, topic = "honest")
|
14
|
+
@topic = topic
|
15
|
+
@request_key = request_key
|
16
|
+
@worker_class = worker_class
|
17
|
+
@queue_name = queue
|
18
|
+
@durable = durable
|
19
|
+
@subscriber = ::Banter::Server::RabbitMQSubscriber.new(@request_key, @durable, @topic)
|
20
|
+
end
|
21
|
+
|
22
|
+
def start
|
23
|
+
@subscriber.start(@queue_name, false) do |delivery_info, properties, envelope|
|
24
|
+
message_received(delivery_info, properties, envelope)
|
25
|
+
true
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def shutdown
|
30
|
+
@subscriber.teardown if @subscriber.present?
|
31
|
+
# TODO -thl
|
32
|
+
# This the kosher thing to do in ruby?
|
33
|
+
teardown
|
34
|
+
end
|
35
|
+
|
36
|
+
def message_received(delivery_info, properties, envelope)
|
37
|
+
Banter.logger.debug("Message received by listener on #{worker_class.name} with payload: #{envelope[:payload]}")
|
38
|
+
worker = worker_class.new(delivery_info, properties, envelope[:context])
|
39
|
+
worker.perform!(envelope[:payload])
|
40
|
+
rescue => e
|
41
|
+
Banter.logger.error("Failed message for #{worker_class.name}")
|
42
|
+
Airbrake.notify("Failed perform for #{worker_class.name}",
|
43
|
+
params: {
|
44
|
+
info: delivery_info,
|
45
|
+
properties: properties,
|
46
|
+
context: envelope[:context],
|
47
|
+
payload: envelope[:payload] })
|
48
|
+
|
49
|
+
end
|
50
|
+
|
51
|
+
protected
|
52
|
+
def teardown
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,75 @@
|
|
1
|
+
module Banter
|
2
|
+
module Server
|
3
|
+
|
4
|
+
class RabbitMQSubscriber
|
5
|
+
attr_reader :listener
|
6
|
+
attr_reader :exchange
|
7
|
+
attr_reader :channel
|
8
|
+
|
9
|
+
def initialize(routing_key, durable = true, topic="honest")
|
10
|
+
@initial_key = routing_key
|
11
|
+
@durable = durable
|
12
|
+
@topic = topic
|
13
|
+
|
14
|
+
if @initial_key.present?
|
15
|
+
@routing_key = "#{@topic}.#{@initial_key}.#"
|
16
|
+
else
|
17
|
+
@routing_key = "#{@topic}.#"
|
18
|
+
end
|
19
|
+
@logger = ::Banter::Logger.new
|
20
|
+
|
21
|
+
self
|
22
|
+
end
|
23
|
+
|
24
|
+
# name - used to ensure that certain consumers are actually listening to an exchange
|
25
|
+
# pass in a lambda for this method to work. We might only want to expose the content instead of
|
26
|
+
# all 3 chunks.
|
27
|
+
def start(name, blocking=false)
|
28
|
+
@connection = Bunny.new(Configuration.configuration[:connection])
|
29
|
+
begin
|
30
|
+
@connection.start
|
31
|
+
rescue => e
|
32
|
+
Airbrake.notify("RabbitMQ unreachable!", params: { message: e.message }, environment_name: ENV['RAILS_ENV'])
|
33
|
+
raise e
|
34
|
+
end
|
35
|
+
|
36
|
+
@channel = @connection.create_channel
|
37
|
+
@exchange = @channel.topic(@topic, :durable => @durable, :auto_delete => false)
|
38
|
+
|
39
|
+
# FIX!!! -thl
|
40
|
+
# Need to ensure that the ids for a server will be reproducible in case a server
|
41
|
+
# goes down and has to get restarted.
|
42
|
+
if @initial_key.present?
|
43
|
+
@queue = "#{@initial_key}.#{name}"
|
44
|
+
else
|
45
|
+
@queue = "#{name}"
|
46
|
+
end
|
47
|
+
|
48
|
+
queue_arguments = {}
|
49
|
+
queue_arguments["x-dead-letter-exchange"] = Configuration.configuration[:dead_letter] if Configuration.configuration[:dead_letter].present?
|
50
|
+
queue_arguments["x-message-ttl"] = Configuration.default_queue_ttl
|
51
|
+
@listener = @channel.queue(@queue, arguments: queue_arguments).bind(@exchange, routing_key: @routing_key, exclusive: false)
|
52
|
+
|
53
|
+
# Parameters for subscribe that might be useful:
|
54
|
+
# :block=>true - Used for long running consumer applications. (backend servers?)
|
55
|
+
@consumer = @listener.subscribe(consumer_tag: name, block: blocking)
|
56
|
+
|
57
|
+
@consumer.on_delivery do |delivery_info, properties, contents|
|
58
|
+
Banter.logger.debug("Message delivery with contents: #{contents}")
|
59
|
+
if delivery_info[:redelivered]
|
60
|
+
Airbrake.notify("PubSub Message redelivery", params: { info: delivery_info, props: properties, contents: contents }, environment_name: ENV['RAILS_ENV'])
|
61
|
+
end
|
62
|
+
message = ::Banter::Message.new.parse(contents)
|
63
|
+
@logger.log_receive(delivery_info[:routing_key], message)
|
64
|
+
yield delivery_info, properties, message
|
65
|
+
true
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def teardown
|
70
|
+
@consumer.cancel if @consumer.present?
|
71
|
+
@connection.close if @connection.present?
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|