pika_que 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +9 -0
- data/.rspec +2 -0
- data/.travis.yml +4 -0
- data/CODE_OF_CONDUCT.md +49 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +21 -0
- data/README.md +41 -0
- data/Rakefile +6 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/examples/demo.rb +42 -0
- data/examples/demo_delay.rb +41 -0
- data/examples/demo_oneoff.rb +29 -0
- data/examples/demo_priority.rb +52 -0
- data/examples/demo_reporter.rb +19 -0
- data/examples/demo_retry.rb +41 -0
- data/examples/demo_worker.rb +17 -0
- data/examples/dev_worker.rb +19 -0
- data/exe/pika_que +8 -0
- data/lib/active_job/queue_adapters/pika_que_adapter.rb +42 -0
- data/lib/pika_que.rb +44 -0
- data/lib/pika_que/broker.rb +88 -0
- data/lib/pika_que/cli.rb +180 -0
- data/lib/pika_que/codecs/json.rb +22 -0
- data/lib/pika_que/codecs/noop.rb +20 -0
- data/lib/pika_que/codecs/rails.rb +22 -0
- data/lib/pika_que/configuration.rb +110 -0
- data/lib/pika_que/connection.rb +47 -0
- data/lib/pika_que/delay_worker.rb +55 -0
- data/lib/pika_que/errors.rb +5 -0
- data/lib/pika_que/handlers/default_handler.rb +31 -0
- data/lib/pika_que/handlers/delay_handler.rb +124 -0
- data/lib/pika_que/handlers/error_handler.rb +69 -0
- data/lib/pika_que/handlers/retry_handler.rb +186 -0
- data/lib/pika_que/launcher.rb +92 -0
- data/lib/pika_que/logging.rb +33 -0
- data/lib/pika_que/metrics.rb +26 -0
- data/lib/pika_que/metrics/log_metric.rb +23 -0
- data/lib/pika_que/metrics/null_metric.rb +14 -0
- data/lib/pika_que/middleware/active_record.rb +13 -0
- data/lib/pika_que/middleware/chain.rb +90 -0
- data/lib/pika_que/processor.rb +45 -0
- data/lib/pika_que/publisher.rb +23 -0
- data/lib/pika_que/rails.rb +62 -0
- data/lib/pika_que/reporters.rb +18 -0
- data/lib/pika_que/reporters/log_reporter.rb +13 -0
- data/lib/pika_que/runner.rb +24 -0
- data/lib/pika_que/subscriber.rb +80 -0
- data/lib/pika_que/util.rb +17 -0
- data/lib/pika_que/version.rb +3 -0
- data/lib/pika_que/worker.rb +99 -0
- data/pika_que.gemspec +37 -0
- metadata +181 -0
data/exe/pika_que
ADDED
@@ -0,0 +1,42 @@
|
|
1
|
+
require 'pika_que'
|
2
|
+
require 'pika_que/codecs/rails'
|
3
|
+
require 'thread'
|
4
|
+
|
5
|
+
module ActiveJob
|
6
|
+
module QueueAdapters
|
7
|
+
# == PikaQue adapter for Active Job
|
8
|
+
#
|
9
|
+
# PikaQue is a RabbitMQ background processing framework for Ruby.
|
10
|
+
#
|
11
|
+
# Read more about PikaQue {here}[https://github.com/dwkoogt/pika_que].
|
12
|
+
#
|
13
|
+
# To use PikaQue set the queue_adapter config to +:pika_que+.
|
14
|
+
#
|
15
|
+
# Rails.application.config.active_job.queue_adapter = :pika_que
|
16
|
+
#
|
17
|
+
class PikaQueAdapter
|
18
|
+
@monitor = Monitor.new
|
19
|
+
|
20
|
+
class << self
|
21
|
+
def enqueue(job) #:nodoc:
|
22
|
+
@monitor.synchronize do
|
23
|
+
JobWrapper.enqueue job.serialize, to_queue: job.queue_name
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def enqueue_at(job, timestamp) #:nodoc:
|
28
|
+
@monitor.synchronize do
|
29
|
+
JobWrapper.enqueue_at job.serialize, timestamp, routing_key: job.queue_name
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
class JobWrapper #:nodoc:
|
35
|
+
extend PikaQue::Worker::ClassMethods
|
36
|
+
config codec: PikaQue::Codecs::RAILS
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
autoload :PikaQueAdapter
|
41
|
+
end
|
42
|
+
end
|
data/lib/pika_que.rb
ADDED
@@ -0,0 +1,44 @@
|
|
1
|
+
require 'bunny'
|
2
|
+
require 'concurrent/executors'
|
3
|
+
|
4
|
+
require 'pika_que/configuration'
|
5
|
+
require 'pika_que/errors'
|
6
|
+
require 'pika_que/logging'
|
7
|
+
require 'pika_que/version'
|
8
|
+
|
9
|
+
require 'pika_que/connection'
|
10
|
+
require 'pika_que/publisher'
|
11
|
+
require 'pika_que/reporters/log_reporter'
|
12
|
+
require 'pika_que/middleware/chain'
|
13
|
+
require 'pika_que/worker'
|
14
|
+
|
15
|
+
module PikaQue
|
16
|
+
|
17
|
+
def self.config
|
18
|
+
@config ||= Configuration.new
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.logger
|
22
|
+
PikaQue::Logging.logger
|
23
|
+
end
|
24
|
+
|
25
|
+
def self.logger=(logger)
|
26
|
+
PikaQue::Logging.logger = logger
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.connection
|
30
|
+
@connection ||= Connection.create
|
31
|
+
end
|
32
|
+
|
33
|
+
def self.middleware
|
34
|
+
@chain ||= Middleware::Chain.new
|
35
|
+
yield @chain if block_given?
|
36
|
+
@chain
|
37
|
+
end
|
38
|
+
|
39
|
+
def self.reporters
|
40
|
+
config[:reporters] << PikaQue::Reporters::LogReporter.new if config[:reporters].empty?
|
41
|
+
config[:reporters]
|
42
|
+
end
|
43
|
+
|
44
|
+
end
|
@@ -0,0 +1,88 @@
|
|
1
|
+
require 'pika_que/util'
|
2
|
+
require 'pika_que/handlers/default_handler'
|
3
|
+
|
4
|
+
module PikaQue
|
5
|
+
class Broker
|
6
|
+
|
7
|
+
def initialize(processor = nil, opts = {})
|
8
|
+
@opts = PikaQue.config.merge(opts)
|
9
|
+
@processor = processor
|
10
|
+
@handlers = {}
|
11
|
+
end
|
12
|
+
|
13
|
+
def start
|
14
|
+
@connection ||= @opts[:connection_options] ? PikaQue::Connection.create(@opts[:connection_options]) : PikaQue.connection
|
15
|
+
@connection.ensure_connection
|
16
|
+
end
|
17
|
+
|
18
|
+
def stop
|
19
|
+
@connection.disconnect! if local_connection?
|
20
|
+
end
|
21
|
+
|
22
|
+
def local_connection?
|
23
|
+
@opts[:connection_options] || @processor.nil?
|
24
|
+
end
|
25
|
+
|
26
|
+
def queue(queue_name, queue_opts = {})
|
27
|
+
begin
|
28
|
+
queue = channel.queue(queue_name, queue_opts)
|
29
|
+
routing_key = queue_opts[:routing_key] || queue_name
|
30
|
+
routing_keys = [routing_key, *queue_opts[:routing_keys]]
|
31
|
+
|
32
|
+
routing_keys.each do |key|
|
33
|
+
queue.bind(exchange, routing_key: key)
|
34
|
+
end
|
35
|
+
queue
|
36
|
+
rescue => e
|
37
|
+
PikaQue.logger.fatal e.message
|
38
|
+
raise SetupError.new e.message
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def handler(handler_class, handler_opts = {})
|
43
|
+
if handler_class
|
44
|
+
h_key = "#{handler_class}-#{handler_opts.hash}"
|
45
|
+
_handler = @handlers[h_key]
|
46
|
+
unless _handler
|
47
|
+
_handler = handler_class.new(handler_opts.merge({ connection: @connection }))
|
48
|
+
@handlers[h_key] = _handler
|
49
|
+
end
|
50
|
+
_handler
|
51
|
+
else
|
52
|
+
default_handler
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
def default_handler
|
57
|
+
@default_handler ||= @opts[:handler_class] ? PikaQue::Util.constantize(@opts[:handler_class]).new(@opts[:handler_options].merge({ connection: @connection })) : PikaQue::Handlers::DefaultHandler.new
|
58
|
+
end
|
59
|
+
|
60
|
+
def exchange
|
61
|
+
@exchange ||= channel.exchange(@opts[:exchange], @opts[:exchange_options])
|
62
|
+
end
|
63
|
+
|
64
|
+
def channel
|
65
|
+
@channel ||= init_channel
|
66
|
+
end
|
67
|
+
|
68
|
+
def init_channel
|
69
|
+
@connection.create_channel(nil, @opts[:channel_options][:consumer_pool_size]).tap do |ch|
|
70
|
+
ch.prefetch(@opts[:channel_options][:prefetch])
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
def cleanup(force = false)
|
75
|
+
if (@processor && force) || !@processor
|
76
|
+
@channel.close unless @channel.closed?
|
77
|
+
@channel = nil
|
78
|
+
@exchange = nil
|
79
|
+
if @default_handler
|
80
|
+
@default_handler.close
|
81
|
+
@default_handler = nil
|
82
|
+
end
|
83
|
+
@handlers.values.each(&:close)
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
end
|
88
|
+
end
|
data/lib/pika_que/cli.rb
ADDED
@@ -0,0 +1,180 @@
|
|
1
|
+
require 'optparse'
|
2
|
+
|
3
|
+
require 'pika_que/connection'
|
4
|
+
require 'pika_que/launcher'
|
5
|
+
require 'pika_que/runner'
|
6
|
+
|
7
|
+
module PikaQue
|
8
|
+
class CLI
|
9
|
+
|
10
|
+
attr_accessor :environment
|
11
|
+
|
12
|
+
def parse(args = ARGV)
|
13
|
+
opts = parse_options(args)
|
14
|
+
config.merge!(opts)
|
15
|
+
init_logger
|
16
|
+
daemonize
|
17
|
+
write_pid
|
18
|
+
end
|
19
|
+
|
20
|
+
def run
|
21
|
+
|
22
|
+
load_app
|
23
|
+
|
24
|
+
PikaQue.middleware
|
25
|
+
|
26
|
+
runner = Runner.new
|
27
|
+
|
28
|
+
begin
|
29
|
+
|
30
|
+
Launcher.launch(runner) do
|
31
|
+
runner.run
|
32
|
+
end
|
33
|
+
|
34
|
+
exit 0
|
35
|
+
rescue Interrupt, SetupError => e
|
36
|
+
PikaQue.logger.info "Shutting down: #{e.class.name} received"
|
37
|
+
runner.stop
|
38
|
+
exit 1
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def config
|
43
|
+
PikaQue.config
|
44
|
+
end
|
45
|
+
|
46
|
+
def init_logger
|
47
|
+
PikaQue::Logging.init_logger(config[:logfile]) if config[:logfile]
|
48
|
+
PikaQue.logger.level = ::Logger::WARN if config[:quite]
|
49
|
+
PikaQue.logger.level = ::Logger::DEBUG if config[:verbose]
|
50
|
+
end
|
51
|
+
|
52
|
+
def daemonize
|
53
|
+
return unless config[:daemon]
|
54
|
+
|
55
|
+
files_to_reopen = []
|
56
|
+
ObjectSpace.each_object(File) do |file|
|
57
|
+
files_to_reopen << file unless file.closed?
|
58
|
+
end
|
59
|
+
|
60
|
+
::Process.daemon(true, true)
|
61
|
+
|
62
|
+
files_to_reopen.each do |file|
|
63
|
+
begin
|
64
|
+
file.reopen file.path, "a+"
|
65
|
+
file.sync = true
|
66
|
+
rescue ::Exception
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
[$stdout, $stderr].each do |io|
|
71
|
+
File.open(config[:logfile], 'ab') do |f|
|
72
|
+
io.reopen(f)
|
73
|
+
end
|
74
|
+
io.sync = true
|
75
|
+
end
|
76
|
+
$stdin.reopen(File::NULL)
|
77
|
+
|
78
|
+
init_logger
|
79
|
+
end
|
80
|
+
|
81
|
+
def write_pid
|
82
|
+
if path = config[:pidfile]
|
83
|
+
pidfile = File.expand_path(path)
|
84
|
+
File.open(pidfile, 'w') do |f|
|
85
|
+
f.puts ::Process.pid
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
def load_app
|
91
|
+
if File.directory?(config[:require])
|
92
|
+
rails_path = File.expand_path(File.join(config[:require], 'config', 'environment.rb'))
|
93
|
+
if File.exist?(rails_path)
|
94
|
+
ENV['RACK_ENV'] = ENV['RAILS_ENV'] = environment
|
95
|
+
PikaQue.logger.info "found rails project (#{config[:require]}), booting app in #{ENV['RACK_ENV']} environment"
|
96
|
+
require 'rails'
|
97
|
+
require 'pika_que/rails'
|
98
|
+
require rails_path
|
99
|
+
::Rails.application.eager_load!
|
100
|
+
end
|
101
|
+
else
|
102
|
+
require(File.expand_path(config[:require])) || raise(ArgumentError, 'require returned false')
|
103
|
+
end
|
104
|
+
|
105
|
+
if config[:delay]
|
106
|
+
config.add_processor(config.delete(:delay_options))
|
107
|
+
else
|
108
|
+
config.delete(:delay_options)
|
109
|
+
end
|
110
|
+
|
111
|
+
if config[:workers]
|
112
|
+
config.add_processor({ workers: config.delete(:workers) })
|
113
|
+
end
|
114
|
+
|
115
|
+
end
|
116
|
+
|
117
|
+
def parse_options(args)
|
118
|
+
opts = {}
|
119
|
+
|
120
|
+
@parser = OptionParser.new do |o|
|
121
|
+
o.banner = 'usage: pika_que [options]'
|
122
|
+
|
123
|
+
o.on '-c', '--concurrency INT', "processor threads to use" do |arg|
|
124
|
+
opts[:concurrency] = Integer(arg)
|
125
|
+
end
|
126
|
+
|
127
|
+
o.on '-d', '--daemon', "Daemonize process" do |arg|
|
128
|
+
opts[:daemon] = arg
|
129
|
+
end
|
130
|
+
|
131
|
+
o.on '-e', '--environment ENV', "Application environment" do |arg|
|
132
|
+
opts[:environment] = arg
|
133
|
+
end
|
134
|
+
|
135
|
+
o.on '-q', '--quite', "Print quite output" do |arg|
|
136
|
+
opts[:quite] = arg
|
137
|
+
end
|
138
|
+
|
139
|
+
o.on '-v', '--verbose', "Print verbose output" do |arg|
|
140
|
+
opts[:verbose] = arg
|
141
|
+
end
|
142
|
+
|
143
|
+
o.on '-r', '--require [PATH|DIR]', "Location of Rails application with workers or file to require" do |arg|
|
144
|
+
opts[:require] = arg
|
145
|
+
end
|
146
|
+
|
147
|
+
o.on '-w', '--worker WORKER(S)', "comma separated list of workers" do |arg|
|
148
|
+
opts[:workers] = arg.split(",")
|
149
|
+
end
|
150
|
+
|
151
|
+
o.on '-L', '--logfile PATH', "path to writable logfile" do |arg|
|
152
|
+
opts[:logfile] = arg
|
153
|
+
end
|
154
|
+
|
155
|
+
o.on '-P', '--pidfile PATH', "path to pidfile" do |arg|
|
156
|
+
opts[:pidfile] = arg
|
157
|
+
end
|
158
|
+
|
159
|
+
o.on '-V', '--version', "Print version and exit" do
|
160
|
+
puts "PikaQue #{PikaQue::VERSION}"
|
161
|
+
exit 0
|
162
|
+
end
|
163
|
+
|
164
|
+
o.on_tail '-h', '--help', 'Show this message and exit' do
|
165
|
+
puts o
|
166
|
+
exit 0
|
167
|
+
end
|
168
|
+
end
|
169
|
+
|
170
|
+
@parser.parse!(args)
|
171
|
+
|
172
|
+
@environment = opts[:environment] || ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
|
173
|
+
|
174
|
+
opts[:logfile] ||= opts[:daemon] ? 'pika_que.log' : STDOUT
|
175
|
+
|
176
|
+
opts
|
177
|
+
end
|
178
|
+
|
179
|
+
end
|
180
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require 'json'
|
2
|
+
|
3
|
+
module PikaQue
|
4
|
+
module Codecs
|
5
|
+
module JSON
|
6
|
+
extend self
|
7
|
+
|
8
|
+
def encode(payload)
|
9
|
+
::JSON.generate(payload, quirks_mode: true)
|
10
|
+
end
|
11
|
+
|
12
|
+
def decode(payload)
|
13
|
+
::JSON.parse(payload, quirks_mode: true)
|
14
|
+
end
|
15
|
+
|
16
|
+
def content_type
|
17
|
+
'application/json'
|
18
|
+
end
|
19
|
+
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require 'active_support/json'
|
2
|
+
|
3
|
+
module PikaQue
|
4
|
+
module Codecs
|
5
|
+
module RAILS
|
6
|
+
extend self
|
7
|
+
|
8
|
+
def encode(payload)
|
9
|
+
::ActiveSupport::JSON.encode(payload)
|
10
|
+
end
|
11
|
+
|
12
|
+
def decode(payload)
|
13
|
+
::ActiveSupport::JSON.decode(payload)
|
14
|
+
end
|
15
|
+
|
16
|
+
def content_type
|
17
|
+
'application/json'
|
18
|
+
end
|
19
|
+
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,110 @@
|
|
1
|
+
require 'forwardable'
|
2
|
+
|
3
|
+
require 'pika_que/codecs/json'
|
4
|
+
require 'pika_que/codecs/noop'
|
5
|
+
|
6
|
+
require 'pika_que/processor'
|
7
|
+
require 'pika_que/handlers/default_handler'
|
8
|
+
|
9
|
+
require 'pika_que/delay_worker'
|
10
|
+
require 'pika_que/handlers/delay_handler'
|
11
|
+
|
12
|
+
module PikaQue
|
13
|
+
class Configuration
|
14
|
+
extend Forwardable
|
15
|
+
|
16
|
+
def_delegators :@config, :to_hash, :[], :[]=, :==, :fetch, :delete, :has_key?
|
17
|
+
|
18
|
+
EXCHANGE_OPTION_DEFAULTS = {
|
19
|
+
:type => :direct,
|
20
|
+
:durable => true,
|
21
|
+
:auto_delete => false,
|
22
|
+
:arguments => {} # Passed as :arguments to Bunny::Channel#exchange
|
23
|
+
}.freeze
|
24
|
+
|
25
|
+
QUEUE_OPTION_DEFAULTS = {
|
26
|
+
:durable => true,
|
27
|
+
:auto_delete => false,
|
28
|
+
:exclusive => false,
|
29
|
+
:arguments => {}
|
30
|
+
}.freeze
|
31
|
+
|
32
|
+
CHANNEL_OPTION_DEFAULTS = {
|
33
|
+
:consumer_pool_size => 1,
|
34
|
+
:prefetch => 10
|
35
|
+
}.freeze
|
36
|
+
|
37
|
+
DELAY_PROCESSOR_DEFAULTS = {
|
38
|
+
:workers => [PikaQue::DelayWorker],
|
39
|
+
:handler_class => PikaQue::Handlers::DelayHandler,
|
40
|
+
:concurrency => 1
|
41
|
+
}.freeze
|
42
|
+
|
43
|
+
DEFAULT_CONFIG = {
|
44
|
+
:exchange => 'pika-que',
|
45
|
+
:heartbeat => 30,
|
46
|
+
:channel_options => CHANNEL_OPTION_DEFAULTS,
|
47
|
+
:exchange_options => EXCHANGE_OPTION_DEFAULTS,
|
48
|
+
:queue_options => QUEUE_OPTION_DEFAULTS,
|
49
|
+
:concurrency => 1,
|
50
|
+
:ack => true,
|
51
|
+
:handler_class => PikaQue::Handlers::DefaultHandler,
|
52
|
+
:handler_options => {},
|
53
|
+
:codec => PikaQue::Codecs::JSON,
|
54
|
+
:processors => [],
|
55
|
+
:reporters => [],
|
56
|
+
:metrics => nil,
|
57
|
+
:delay => true,
|
58
|
+
:delay_options => DELAY_PROCESSOR_DEFAULTS,
|
59
|
+
:pidfile => nil,
|
60
|
+
:require => '.'
|
61
|
+
}.freeze
|
62
|
+
|
63
|
+
# processor example
|
64
|
+
# @processor Processor class
|
65
|
+
# @workers array of worker classes
|
66
|
+
# @connection connection params if using separate connection
|
67
|
+
# {
|
68
|
+
# :processor => Processor,
|
69
|
+
# :connection_options => {},
|
70
|
+
# :workers => [],
|
71
|
+
# :concurrency => 1,
|
72
|
+
# :ack => true,
|
73
|
+
# :handler_class => nil,
|
74
|
+
# :handler_options => nil,
|
75
|
+
# :codec => PikaQue::Codecs::JSON
|
76
|
+
# }
|
77
|
+
|
78
|
+
def initialize
|
79
|
+
@config = DEFAULT_CONFIG.dup
|
80
|
+
end
|
81
|
+
|
82
|
+
def merge!(other = {})
|
83
|
+
@config = deep_merge(@config, other)
|
84
|
+
end
|
85
|
+
|
86
|
+
def merge(other = {})
|
87
|
+
instance = self.class.new
|
88
|
+
instance.merge! to_hash
|
89
|
+
instance.merge! other
|
90
|
+
instance
|
91
|
+
end
|
92
|
+
|
93
|
+
def deep_merge(first, second)
|
94
|
+
merger = proc { |_, v1, v2| Hash === v1 && Hash === v2 ? v1.merge(v2, &merger) : v2 }
|
95
|
+
first.merge(second, &merger)
|
96
|
+
end
|
97
|
+
|
98
|
+
def processor(opts = {})
|
99
|
+
{
|
100
|
+
:processor => PikaQue::Processor,
|
101
|
+
:workers => []
|
102
|
+
}.merge(opts)
|
103
|
+
end
|
104
|
+
|
105
|
+
def add_processor(opts = {})
|
106
|
+
@config[:processors] << processor(opts)
|
107
|
+
end
|
108
|
+
|
109
|
+
end
|
110
|
+
end
|