rabbitek 0.1.1

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,91 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rabbitek
4
+ ##
5
+ # Consumer helpers
6
+ module Consumer
7
+ def self.included(base)
8
+ base.extend(ClassMethods)
9
+ end
10
+
11
+ attr_reader :channel, :queue, :retry_or_delayed_queue, :retry_or_delayed_exchange
12
+
13
+ def initialize(channel, queue, retry_or_delayed_queue, retry_or_delayed_exchange)
14
+ @channel = channel
15
+ @queue = queue
16
+ @retry_or_delayed_queue = retry_or_delayed_queue
17
+ @retry_or_delayed_exchange = retry_or_delayed_exchange
18
+ end
19
+
20
+ def ack!(delivery_info, multiple = false)
21
+ channel.ack(delivery_info.delivery_tag, multiple)
22
+ end
23
+
24
+ def logger
25
+ Rabbitek.logger
26
+ end
27
+
28
+ def parse_message(message)
29
+ Utils::Oj.load(message)
30
+ end
31
+
32
+ def perform(*_args)
33
+ raise NotImplementedError
34
+ end
35
+
36
+ def set_context
37
+ Thread.current[:rabbit_context] = { consumer: self.class.name, queue: @queue.name, job_id: SecureRandom.uuid }
38
+ end
39
+
40
+ def jid
41
+ Thread.current[:rabbit_context][:job_id]
42
+ end
43
+
44
+ module ClassMethods # rubocop:disable Style/Documentation
45
+ attr_accessor :rabbit_options_hash
46
+
47
+ def rabbit_options(opts)
48
+ self.rabbit_options_hash = default_rabbit_options(opts).with_indifferent_access.merge(opts)
49
+ end
50
+
51
+ def perform_async(payload, opts: {}, channel: nil)
52
+ publisher = Publisher.new(
53
+ rabbit_options_hash[:bind_exchange],
54
+ exchange_type: rabbit_options_hash[:bind_exchange_type],
55
+ channel: channel
56
+ )
57
+ publish_with_publisher(publisher, payload, opts)
58
+ ensure
59
+ publisher&.close unless channel
60
+ end
61
+
62
+ def perform_in(time, payload, opts: {}, channel: nil)
63
+ publisher = Publisher.new(
64
+ Utils::RabbitObjectNames.retry_or_delayed_bind_exchange(rabbit_options_hash[:bind_exchange]),
65
+ exchange_type: :fanout,
66
+ channel: channel
67
+ )
68
+ publish_with_publisher(publisher, payload, {
69
+ expiration: time.to_i * 1000, # in milliseconds
70
+ headers: { 'x-dead-letter-routing-key': to_s }
71
+ }.merge(opts))
72
+ ensure
73
+ publisher&.close unless channel
74
+ end
75
+
76
+ def perform_at(at_time, payload, opts: {}, channel: nil)
77
+ perform_in(at_time - Time.current, payload, opts: opts, channel: channel)
78
+ end
79
+
80
+ def publish_with_publisher(publisher, payload, opts)
81
+ publisher.publish(payload, { routing_key: to_s }.merge(opts))
82
+ end
83
+
84
+ private
85
+
86
+ def default_rabbit_options(opts)
87
+ YAML.load_file(opts[:config_file]).with_indifferent_access[:parameters]
88
+ end
89
+ end
90
+ end
91
+ end
@@ -0,0 +1,34 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative '../server_hook'
4
+
5
+ module Rabbitek
6
+ module Server
7
+ module Hooks
8
+ ##
9
+ # OpenTracing server hook
10
+ class OpenTracing < Rabbitek::ServerHook
11
+ def call(consumer, delivery_info, properties, payload)
12
+ response = nil
13
+
14
+ ::OpenTracing.start_active_span(delivery_info.routing_key, opts(delivery_info, properties)) do |scope|
15
+ begin
16
+ response = super
17
+ rescue StandardError => e
18
+ Utils::OpenTracing.log_error(scope.span, e)
19
+ raise
20
+ end
21
+ end
22
+
23
+ response
24
+ end
25
+
26
+ private
27
+
28
+ def opts(delivery_info, properties)
29
+ Utils::OpenTracing.server_options(delivery_info, properties)
30
+ end
31
+ end
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,70 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative '../server_hook'
4
+
5
+ module Rabbitek
6
+ module Server
7
+ module Hooks
8
+ ##
9
+ # Hook to retry failed jobs
10
+ class Retry < Rabbitek::ServerHook
11
+ include Loggable
12
+
13
+ def call(consumer, delivery_info, properties, payload)
14
+ super
15
+ rescue StandardError
16
+ retry_message(consumer, payload, delivery_info, properties)
17
+ raise
18
+ end
19
+
20
+ private
21
+
22
+ def retry_message(consumer, payload, delivery_info, properties)
23
+ headers = properties.headers || {}
24
+ dead_headers = headers.fetch('x-death', []).last || {}
25
+
26
+ retry_count = headers.fetch('x-retry-count', 0)
27
+ expiration = dead_headers.fetch('original-expiration', 1000).to_i
28
+
29
+ warn_log(retry_count, expiration, consumer)
30
+
31
+ # acknowledge existing message
32
+ consumer.ack!(delivery_info)
33
+
34
+ if retry_count <= 25
35
+ # Set the new expiration with an increasing factor
36
+ new_expiration = expiration * 1.5
37
+
38
+ # Publish to retry queue with new expiration
39
+ publish_to_retry_queue(consumer, new_expiration, delivery_info, payload, retry_count)
40
+ else
41
+ publish_to_dead_queue
42
+ end
43
+ end
44
+
45
+ def warn_log(retry_count, expiration, consumer)
46
+ warn(
47
+ message: 'Failure!',
48
+ retry_count: retry_count,
49
+ expiration: expiration,
50
+ consumer: consumer.class.to_s,
51
+ jid: consumer.jid
52
+ )
53
+ end
54
+
55
+ def publish_to_retry_queue(consumer, new_expiration, delivery_info, payload, retry_count)
56
+ consumer.retry_or_delayed_exchange.publish(
57
+ payload,
58
+ expiration: new_expiration.to_i,
59
+ routing_key: delivery_info.routing_key,
60
+ headers: { 'x-retry-count': retry_count + 1, 'x-dead-letter-routing-key': delivery_info.routing_key }
61
+ )
62
+ end
63
+
64
+ def publish_to_dead_queue
65
+ # TODO: implement dead queue
66
+ end
67
+ end
68
+ end
69
+ end
70
+ end
@@ -0,0 +1,30 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative '../server_hook'
4
+
5
+ module Rabbitek
6
+ module Server
7
+ module Hooks
8
+ ##
9
+ # Hook to keep track of time used for processing single job
10
+ class TimeTracker < Rabbitek::ServerHook
11
+ include Loggable
12
+
13
+ def call(consumer, delivery_info, properties, payload)
14
+ info(message: 'Starting', consumer: delivery_info.routing_key, jid: consumer.jid)
15
+
16
+ start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
17
+
18
+ super
19
+ ensure
20
+ info(
21
+ message: 'Finished',
22
+ consumer: delivery_info.routing_key,
23
+ time: Process.clock_gettime(Process::CLOCK_MONOTONIC) - start,
24
+ jid: consumer.jid
25
+ )
26
+ end
27
+ end
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,11 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rabbitek
4
+ ##
5
+ # Base server hook class
6
+ class ServerHook
7
+ def call(consumer, delivery_info, properties, payload)
8
+ yield(consumer, delivery_info, properties, payload)
9
+ end
10
+ end
11
+ end
@@ -0,0 +1,92 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rabbitek
4
+ ##
5
+ # Main server startup
6
+ class Starter
7
+ include Loggable
8
+
9
+ def initialize(connection, configuration)
10
+ @connection = connection
11
+ @queue_name = configuration[:parameters][:queue]
12
+ @consumers = configuration[:consumers]
13
+ @opts = configuration[:parameters]
14
+ end
15
+
16
+ def start
17
+ setup_bindings!
18
+
19
+ work_queue.subscribe(manual_ack: true) do |delivery_info, properties, payload|
20
+ on_message_received(delivery_info, properties, payload)
21
+ end
22
+ end
23
+
24
+ private
25
+
26
+ attr_reader :connection, :queue_name, :consumers, :opts
27
+
28
+ def setup_bindings!
29
+ consumers.each do |worker_class|
30
+ work_queue.bind(work_exchange, routing_key: worker_class.to_s)
31
+ end
32
+ retry_or_delayed_queue.bind(retry_or_delayed_exchange)
33
+ end
34
+
35
+ def on_message_received(delivery_info, properties, payload)
36
+ consumer = consumer_instance(delivery_info.routing_key)
37
+ consumer.set_context
38
+
39
+ hook_walker = Utils::HookWalker.new(Rabbitek.config.server_hooks)
40
+
41
+ hook_walker.call!(consumer, delivery_info, properties, payload) do |*args|
42
+ run_job(*args)
43
+ end
44
+ end
45
+
46
+ def run_job(consumer, delivery_info, properties, payload)
47
+ consumer.perform(consumer.parse_message(payload), delivery_info, properties)
48
+ rescue StandardError => e
49
+ error(message: e.inspect, backtrace: e.backtrace, consumer: consumer.class, jid: consumer.jid)
50
+ end
51
+
52
+ def consumer_instance(routing_key)
53
+ Thread.current[:worker_classes] ||= {}
54
+ klass = Thread.current[:worker_classes][routing_key] ||= routing_key.constantize
55
+ klass.new(channel, work_queue, retry_or_delayed_queue, retry_or_delayed_exchange)
56
+ rescue NameError
57
+ nil # TODO: to dead queue
58
+ end
59
+
60
+ def channel
61
+ @channel ||= begin
62
+ channel = connection.create_channel
63
+ channel.basic_qos(opts[:basic_qos]) if opts[:basic_qos].present?
64
+ channel
65
+ end
66
+ end
67
+
68
+ def work_exchange
69
+ @work_exchange ||= Utils::Common.exchange(channel, 'direct', opts[:bind_exchange])
70
+ end
71
+
72
+ def work_queue
73
+ @work_queue ||= Utils::Common.queue(channel, queue_name, opts[:queue_attributes])
74
+ end
75
+
76
+ def retry_or_delayed_queue
77
+ @retry_or_delayed_queue ||= Utils::Common.queue(
78
+ channel,
79
+ Utils::RabbitObjectNames.retry_or_delayed_queue(opts[:queue]),
80
+ arguments: { 'x-dead-letter-exchange': opts[:bind_exchange] }
81
+ )
82
+ end
83
+
84
+ def retry_or_delayed_exchange
85
+ @retry_or_delayed_exchange ||= Utils::Common.exchange(
86
+ channel,
87
+ :fanout,
88
+ Utils::RabbitObjectNames.retry_or_delayed_bind_exchange(opts[:bind_exchange])
89
+ )
90
+ end
91
+ end
92
+ end
@@ -0,0 +1,30 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rabbitek
4
+ module Utils
5
+ ##
6
+ # Common utilities to create/use RabbitMQ exchange or queue
7
+ class Common
8
+ class << self
9
+ def exchange(channel, exchange_type, exchange_name)
10
+ channel.public_send(exchange_type || 'direct', exchange_name, durable: true, auto_delete: false)
11
+ end
12
+
13
+ def queue(channel, name, opts)
14
+ opts ||= {}
15
+ opts = symbolize_keys(opts.to_hash)
16
+ opts[:durable] = true
17
+ opts[:auto_delete] = false
18
+
19
+ channel.queue(name, opts)
20
+ end
21
+
22
+ private
23
+
24
+ def symbolize_keys(hash)
25
+ hash.each_with_object({}) { |(k, v), memo| memo[k.to_sym] = v; }
26
+ end
27
+ end
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,36 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rabbitek
4
+ module Utils
5
+ ##
6
+ # Utility to work down the hooks setup
7
+ class HookWalker
8
+ include Loggable
9
+
10
+ def initialize(hooks = [])
11
+ @hooks = hooks.clone
12
+ end
13
+
14
+ def call!(*args)
15
+ return yield(*args) unless hooks.any?
16
+ hook = hooks.pop
17
+
18
+ debug "Calling hook: #{hook.class}"
19
+
20
+ begin
21
+ return_args = hook.call(*args) do |*new_args|
22
+ hooks.any? ? call!(*new_args) { |*next_args| yield(*next_args) } : yield(*new_args)
23
+ end
24
+ ensure
25
+ debug "Finishing hook: #{hook.class}"
26
+ end
27
+
28
+ return_args
29
+ end
30
+
31
+ private
32
+
33
+ attr_reader :hooks
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,17 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rabbitek
4
+ module Utils
5
+ ##
6
+ # Oj methods wrapper
7
+ class Oj
8
+ def self.dump(obj)
9
+ ::Oj.dump(obj, mode: :compat)
10
+ end
11
+
12
+ def self.load(string)
13
+ ::Oj.load(string, mode: :compat)
14
+ end
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,67 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rabbitek
4
+ module Utils
5
+ ##
6
+ # OpenTracing helpers
7
+ class OpenTracing
8
+ OPENTRACING_COMPONENT = 'rabbitek'
9
+ OPENTRACING_KIND_SERVER = 'server'
10
+ OPENTRACING_KIND_CLIENT = 'client'
11
+
12
+ class << self
13
+ def inject!(span, carrier)
14
+ ::OpenTracing.inject(span.context, ::OpenTracing::FORMAT_TEXT_MAP, carrier)
15
+ end
16
+
17
+ def client_options(params)
18
+ {
19
+ tags: {
20
+ 'component' => OPENTRACING_COMPONENT,
21
+ 'span.kind' => OPENTRACING_KIND_CLIENT,
22
+ 'rabbitmq.routing_key' => params[:routing_key]
23
+ }
24
+ }
25
+ end
26
+
27
+ def server_options(delivery_info, properties)
28
+ references = server_references(properties)
29
+
30
+ options = {
31
+ tags: {
32
+ 'component' => OPENTRACING_COMPONENT,
33
+ 'span.kind' => OPENTRACING_KIND_SERVER,
34
+ 'rabbitmq.routing_key' => delivery_info.routing_key,
35
+ 'rabbitmq.jid' => Thread.current[:rabbit_context][:jid],
36
+ 'rabbitmq.queue' => Thread.current[:rabbit_context][:queue],
37
+ 'rabbitmq.worker' => Thread.current[:rabbit_context][:consumer]
38
+ }
39
+ }
40
+
41
+ options[:references] = [references] if references
42
+ options
43
+ end
44
+
45
+ def log_error(span, err)
46
+ span.set_tag('error', true)
47
+ span.log_kv(
48
+ event: 'error',
49
+ 'error.kind': err.class.to_s,
50
+ 'error.object': err,
51
+ message: err.message
52
+ )
53
+ end
54
+
55
+ private
56
+
57
+ def server_references(message_properties)
58
+ ::OpenTracing::Reference.follows_from(extract(message_properties))
59
+ end
60
+
61
+ def extract(message_properties)
62
+ ::OpenTracing.extract(::OpenTracing::FORMAT_TEXT_MAP, message_properties.headers)
63
+ end
64
+ end
65
+ end
66
+ end
67
+ end
@@ -0,0 +1,19 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rabbitek
4
+ module Utils
5
+ ##
6
+ # Names builder for exchanges, queues, etc.
7
+ class RabbitObjectNames
8
+ class << self
9
+ def retry_or_delayed_bind_exchange(bind_exchange)
10
+ "#{bind_exchange}.rabbitek.__rod__"
11
+ end
12
+
13
+ def retry_or_delayed_queue(queue_name)
14
+ "#{queue_name}.rabbitek.__rod__"
15
+ end
16
+ end
17
+ end
18
+ end
19
+ end
@@ -0,0 +1,5 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rabbitek
4
+ VERSION = '0.1.1'
5
+ end
data/lib/rabbitek.rb ADDED
@@ -0,0 +1,45 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'rabbitek/version'
4
+
5
+ require 'bunny'
6
+ require 'oj'
7
+ require 'opentracing'
8
+ require 'logger'
9
+
10
+ # active_support
11
+ require 'active_support/core_ext/hash/indifferent_access'
12
+ require 'active_support/core_ext/string/inflections'
13
+
14
+ current_dir = File.dirname(__FILE__)
15
+
16
+ Dir.glob("#{current_dir}/rabbitek/*.rb").each { |file| require file }
17
+ Dir.glob("#{current_dir}/rabbitek/**/*.rb").each { |file| require file }
18
+
19
+ ##
20
+ # High performance background job processing using RabbitMQ
21
+ module Rabbitek
22
+ def self.config
23
+ @config ||= Config.new
24
+ end
25
+
26
+ def self.configure
27
+ yield(config)
28
+ end
29
+
30
+ def self.logger
31
+ @logger ||= Logger.new(STDOUT)
32
+ end
33
+
34
+ def self.create_channel
35
+ bunny_connection.create_channel
36
+ end
37
+
38
+ def self.close_bunny_connection
39
+ bunny_connection.close
40
+ end
41
+
42
+ def self.bunny_connection
43
+ @bunny_connection ||= BunnyConnection.initialize_connection
44
+ end
45
+ end
data/rabbitek.gemspec ADDED
@@ -0,0 +1,36 @@
1
+ # frozen_string_literal: true
2
+
3
+ lib = File.expand_path('lib', __dir__)
4
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
5
+ require 'rabbitek/version'
6
+
7
+ Gem::Specification.new do |spec|
8
+ spec.name = 'rabbitek'
9
+ spec.version = Rabbitek::VERSION
10
+ spec.authors = ['Boostcom']
11
+ spec.email = ['jakub.kruczek@boostcom.no']
12
+
13
+ spec.summary = 'High performance background job processing'
14
+ spec.description = 'High performance background job processing'
15
+ spec.homepage = 'http://boostcom.no'
16
+
17
+ # Specify which files should be added to the gem when it is released.
18
+ # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
19
+ spec.files = Dir.chdir(File.expand_path(__dir__)) do
20
+ `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
21
+ end
22
+ spec.bindir = 'exe'
23
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
24
+ spec.require_paths = ['lib']
25
+
26
+ spec.add_dependency 'activesupport', '> 3.0'
27
+ spec.add_dependency 'bunny', '~> 2.11.0'
28
+ spec.add_dependency 'oj', '~> 3.6'
29
+ spec.add_dependency 'opentracing', '~> 0.4'
30
+ spec.add_dependency 'slop', '~> 4.0'
31
+
32
+ spec.add_development_dependency 'bundler', '~> 1.16'
33
+ spec.add_development_dependency 'rake', '~> 10.0'
34
+ spec.add_development_dependency 'rspec', '~> 3.0'
35
+ spec.add_development_dependency 'rubocop', '~> 0.58.0'
36
+ end