mqjob 0.4.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +51 -0
- data/Gemfile +7 -0
- data/README.md +118 -0
- data/Rakefile +2 -0
- data/bin/console +18 -0
- data/bin/setup +8 -0
- data/lib/mqjob.rb +124 -0
- data/lib/mqjob/thread_pool.rb +44 -0
- data/lib/mqjob/version.rb +3 -0
- data/lib/mqjob/worker.rb +143 -0
- data/lib/mqjob/worker_group.rb +71 -0
- data/lib/plugin.rb +24 -0
- data/lib/plugin/base.rb +15 -0
- data/lib/plugin/pulsar.rb +92 -0
- data/mqjob.gemspec +30 -0
- metadata +100 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: b8e339bb78dcea031b930f7fab3203e5358c0103e4a2c2357b5c321e19fcd311
|
4
|
+
data.tar.gz: eecac402fb51319ff008756bc7ebdf0a005a45b2816b1bf44cb92403726793d4
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 7ed6eb6e2c6642a4edfd0c73a2affc477a0b356150fb7c2d482561340534dd1febf22de73805e52d27948e204275ae188acda3310f3ab1d17ab0899ce253dd0a
|
7
|
+
data.tar.gz: ed2bd2477e35a535aeafc14271ec7b62ae5024e13ec781e56f09b46daf51cd307cb4fd425e0aca6126c6d6246a8e552eaf3fe5a7f589a31a628f8d2cd5e44dde
|
data/.gitignore
ADDED
@@ -0,0 +1,51 @@
|
|
1
|
+
# ---> Ruby
|
2
|
+
*.gem
|
3
|
+
*.rbc
|
4
|
+
/.config
|
5
|
+
/coverage/
|
6
|
+
/InstalledFiles
|
7
|
+
/pkg/
|
8
|
+
/spec/reports/
|
9
|
+
/spec/examples.txt
|
10
|
+
/test/tmp/
|
11
|
+
/test/version_tmp/
|
12
|
+
/tmp/
|
13
|
+
|
14
|
+
# Used by dotenv library to load environment variables.
|
15
|
+
# .env
|
16
|
+
|
17
|
+
## Specific to RubyMotion:
|
18
|
+
.dat*
|
19
|
+
.repl_history
|
20
|
+
build/
|
21
|
+
*.bridgesupport
|
22
|
+
build-iPhoneOS/
|
23
|
+
build-iPhoneSimulator/
|
24
|
+
|
25
|
+
## Specific to RubyMotion (use of CocoaPods):
|
26
|
+
#
|
27
|
+
# We recommend against adding the Pods directory to your .gitignore. However
|
28
|
+
# you should judge for yourself, the pros and cons are mentioned at:
|
29
|
+
# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
|
30
|
+
#
|
31
|
+
# vendor/Pods/
|
32
|
+
|
33
|
+
## Documentation cache and generated files:
|
34
|
+
/.yardoc/
|
35
|
+
/_yardoc/
|
36
|
+
/doc/
|
37
|
+
/rdoc/
|
38
|
+
|
39
|
+
## Environment normalization:
|
40
|
+
/.bundle/
|
41
|
+
/vendor/bundle
|
42
|
+
/lib/bundler/man/
|
43
|
+
|
44
|
+
# for a library or gem, you might want to ignore these files since the code is
|
45
|
+
# intended to run in multiple environments; otherwise, check them in:
|
46
|
+
Gemfile.lock
|
47
|
+
.ruby-version
|
48
|
+
.ruby-gemset
|
49
|
+
|
50
|
+
# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
|
51
|
+
.rvmrc
|
data/Gemfile
ADDED
data/README.md
ADDED
@@ -0,0 +1,118 @@
|
|
1
|
+
# Mqjob
|
2
|
+
|
3
|
+
A fast background processing framework for Ruby using MQ. MQ client was support as plugin. Current only implement Apache Pulsar. Name `Mqjob` was combine MQ and job.
|
4
|
+
|
5
|
+
Inspired By [Sneakers](https://github.com/jondot/sneakers).
|
6
|
+
|
7
|
+
## Examples
|
8
|
+
|
9
|
+
- [initializer](examples/initializers.rb)
|
10
|
+
- [rake task](examples/mqjob.rake)
|
11
|
+
- [Single Job](examples/single_job.rb)
|
12
|
+
- [Multiple Job](examples/multiple_job.rb)
|
13
|
+
- Shell run `WORKERS=SingleJob,MultipleJob bundle exec rake mqjob:run`
|
14
|
+
|
15
|
+
## API
|
16
|
+
|
17
|
+
### Global Config
|
18
|
+
|
19
|
+
using in `Mqjob.configure`.
|
20
|
+
|
21
|
+
- client
|
22
|
+
|
23
|
+
An MQ client instance using in plugin. It should provide `producer` and `consumer` create api.
|
24
|
+
|
25
|
+
- plugin
|
26
|
+
|
27
|
+
Which implementing a specific MQ operations. Must implement basic interface:
|
28
|
+
|
29
|
+
```ruby
|
30
|
+
def listen(topic, worker, opts = {}); end
|
31
|
+
|
32
|
+
def publish(topic, msg, opts = {}); end
|
33
|
+
```
|
34
|
+
|
35
|
+
See [Pulsar](lib/plugin/pulsar.rb) for detail.
|
36
|
+
|
37
|
+
- daemonize
|
38
|
+
|
39
|
+
Config worker run to background.
|
40
|
+
|
41
|
+
- threads
|
42
|
+
|
43
|
+
How many Thread will create for job perform in process. It will init a thread pool.
|
44
|
+
IO-intensive tasks can be appropriately increased, and CPU-intensive tasks can be appropriately reduced.
|
45
|
+
|
46
|
+
- hooks
|
47
|
+
|
48
|
+
config `before_fork` and `after_fork`. NOT implement yet.
|
49
|
+
if you using ActiveRecord, set `wrap_perform` as follow avoid database connection broken.
|
50
|
+
|
51
|
+
```ruby
|
52
|
+
Mqjob.configure do |config|
|
53
|
+
config.hooks = {
|
54
|
+
wrap_perform: lambda {|&b| ActiveRecord::Base.connection_pool.with_connection {b.call}}
|
55
|
+
}
|
56
|
+
end
|
57
|
+
```
|
58
|
+
|
59
|
+
- subscription_mode
|
60
|
+
|
61
|
+
* exclusive
|
62
|
+
|
63
|
+
Same subscription name only one conumser can subscribe to a topic.
|
64
|
+
|
65
|
+
* failover
|
66
|
+
|
67
|
+
All subscribe to a topic only one can receive message, once the subscribe exit the remain pick one keep up.
|
68
|
+
|
69
|
+
* shared
|
70
|
+
|
71
|
+
All subscribe can receive message.
|
72
|
+
|
73
|
+
- logger
|
74
|
+
|
75
|
+
A log instance implement Ruby std logger interface.
|
76
|
+
|
77
|
+
### Worker Config
|
78
|
+
|
79
|
+
- client
|
80
|
+
|
81
|
+
Using difference MQ client for this job. If connect to different types MQ you should config plugin at the same time.
|
82
|
+
|
83
|
+
- plugin
|
84
|
+
|
85
|
+
Using difference MQ client implement.
|
86
|
+
|
87
|
+
- prefetch
|
88
|
+
|
89
|
+
Config how many message per worker pull in one job cycle.
|
90
|
+
|
91
|
+
- subscription_mode
|
92
|
+
|
93
|
+
If set to `exclusive`, should provide subscription_name at the same time.
|
94
|
+
|
95
|
+
- subscription_name
|
96
|
+
|
97
|
+
Config subscription name, effects associated with subscription_mode.
|
98
|
+
|
99
|
+
- logger
|
100
|
+
|
101
|
+
Using difference logger for current worker.
|
102
|
+
|
103
|
+
- topic_type
|
104
|
+
|
105
|
+
* normal
|
106
|
+
|
107
|
+
Ordinary topic name. Default value.
|
108
|
+
|
109
|
+
* regex
|
110
|
+
|
111
|
+
The topic name is a regex, represent topics which match this regex. For example, `persistent://my-tenant/namespace2/topic_*` is all topics in namespace2 that match `/topic_*/`.
|
112
|
+
|
113
|
+
## Note
|
114
|
+
|
115
|
+
- Thread pool will not clean `Thread.current` when thread give back to pool. If you want to use thread storage [RequestStore](https://github.com/steveklabnik/request_store) recommended, it build in support.
|
116
|
+
- database pool should not less than `threads` size.
|
117
|
+
- Maybe you should use connection pool to manage database connection. Like PgBouncer for PostgreSQL, Druid for MySQL.
|
118
|
+
- When topic_type is regex, message enqueue is not supported.
|
data/Rakefile
ADDED
data/bin/console
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "bundler/setup"
|
4
|
+
require "mqjob"
|
5
|
+
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
8
|
+
|
9
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
+
# require "pry"
|
11
|
+
# Pry.start
|
12
|
+
|
13
|
+
Mqjob.configure {
|
14
|
+
|
15
|
+
}
|
16
|
+
|
17
|
+
require "irb"
|
18
|
+
IRB.start(__FILE__)
|
data/bin/setup
ADDED
data/lib/mqjob.rb
ADDED
@@ -0,0 +1,124 @@
|
|
1
|
+
require 'logger'
|
2
|
+
require "mqjob/version"
|
3
|
+
require "mqjob/thread_pool"
|
4
|
+
require 'serverengine'
|
5
|
+
require 'mqjob/worker_group'
|
6
|
+
require 'mqjob/worker'
|
7
|
+
require 'plugin'
|
8
|
+
require 'concurrent/configuration'
|
9
|
+
|
10
|
+
module Mqjob
|
11
|
+
extend self
|
12
|
+
|
13
|
+
attr_reader :config
|
14
|
+
|
15
|
+
def configure(&block)
|
16
|
+
@config ||= Config.new
|
17
|
+
|
18
|
+
yield @config
|
19
|
+
end
|
20
|
+
|
21
|
+
def hooks
|
22
|
+
config&.hooks
|
23
|
+
end
|
24
|
+
|
25
|
+
def default_client
|
26
|
+
config&.client
|
27
|
+
end
|
28
|
+
|
29
|
+
# FIXME when job inherit from parent job it will not appear here!!
|
30
|
+
def registed_class
|
31
|
+
@registed_class ||= []
|
32
|
+
end
|
33
|
+
|
34
|
+
def regist_class(v)
|
35
|
+
@registed_class ||= []
|
36
|
+
@registed_class << v
|
37
|
+
@registed_class.uniq!
|
38
|
+
end
|
39
|
+
|
40
|
+
def logger
|
41
|
+
config.logger ||= ::Logger.new(STDOUT).tap do |logger|
|
42
|
+
logger.formatter = Formatter.new
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
class Config
|
47
|
+
attr_accessor :client,
|
48
|
+
:plugin,
|
49
|
+
:daemonize,
|
50
|
+
:threads,
|
51
|
+
:subscription_mode
|
52
|
+
attr_reader :logger, :hooks
|
53
|
+
|
54
|
+
def initialize(opts = {})
|
55
|
+
@hooks = Hooks.new(opts.delete(:hooks))
|
56
|
+
@plugin = :pulsar
|
57
|
+
|
58
|
+
assign_attributes(opts)
|
59
|
+
|
60
|
+
remove_empty_instance_variables!
|
61
|
+
end
|
62
|
+
|
63
|
+
def hooks=(v)
|
64
|
+
@hooks.update(v)
|
65
|
+
end
|
66
|
+
|
67
|
+
def logger=(v)
|
68
|
+
# fvcking Concurrent::Logging
|
69
|
+
unless v.respond_to?(:call)
|
70
|
+
v.class.class_eval <<-RUBY
|
71
|
+
def call(level, progname, message, &block)
|
72
|
+
add(level, message, progname, &block)
|
73
|
+
end
|
74
|
+
RUBY
|
75
|
+
end
|
76
|
+
@logger = v
|
77
|
+
Concurrent.global_logger = v
|
78
|
+
end
|
79
|
+
|
80
|
+
def assign_attributes(opts)
|
81
|
+
opts.each do |k, v|
|
82
|
+
method = "#{k}="
|
83
|
+
next unless self.respond_to?(method)
|
84
|
+
self.public_send method, v
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
private
|
89
|
+
def remove_empty_instance_variables!
|
90
|
+
instance_variables.each do |x|
|
91
|
+
remove_instance_variable(x) if instance_variable_get(x).nil?
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
class Hooks
|
96
|
+
attr_reader :before_fork, :after_fork, :wrap_perform
|
97
|
+
|
98
|
+
def initialize(opts)
|
99
|
+
update(opts)
|
100
|
+
end
|
101
|
+
|
102
|
+
def update(opts)
|
103
|
+
return if opts.nil? || opts.empty?
|
104
|
+
|
105
|
+
raise "hooks shuld be a Proc map!" unless opts.values.all?{|x| x.nil? || x.is_a?(Proc)}
|
106
|
+
|
107
|
+
@before_fork = opts[:before_fork]
|
108
|
+
@after_fork = opts[:after_fork]
|
109
|
+
@wrap_perform = opts[:wrap_perform]
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
class Formatter < ::Logger::Formatter
|
115
|
+
def call(severity, timestamp, progname, msg)
|
116
|
+
case msg
|
117
|
+
when ::StandardError
|
118
|
+
msg = [msg.message, msg&.backtrace].join(":\n")
|
119
|
+
end
|
120
|
+
|
121
|
+
super
|
122
|
+
end
|
123
|
+
end
|
124
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
require 'concurrent/executors'
|
2
|
+
|
3
|
+
module Mqjob
|
4
|
+
class ThreadPool < ::Concurrent::FixedThreadPool
|
5
|
+
def initialize(num_threads, opts = {})
|
6
|
+
super
|
7
|
+
@job_finish = ConditionVariable.new
|
8
|
+
@job_mutex = Mutex.new
|
9
|
+
end
|
10
|
+
|
11
|
+
# NOTE 使用非缓冲线程池,防止消息丢失
|
12
|
+
def post(*args, &task)
|
13
|
+
wait
|
14
|
+
|
15
|
+
super
|
16
|
+
end
|
17
|
+
|
18
|
+
def ns_execute
|
19
|
+
super
|
20
|
+
|
21
|
+
@job_finish.signal
|
22
|
+
end
|
23
|
+
|
24
|
+
def shutdown
|
25
|
+
super
|
26
|
+
|
27
|
+
@job_finish.broadcast
|
28
|
+
end
|
29
|
+
|
30
|
+
def kill
|
31
|
+
super
|
32
|
+
|
33
|
+
@job_finish.broadcast
|
34
|
+
end
|
35
|
+
|
36
|
+
def wait
|
37
|
+
@job_mutex.synchronize do
|
38
|
+
while running? && (scheduled_task_count - completed_task_count >= max_length)
|
39
|
+
@job_finish.wait(@job_mutex, 0.05)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
data/lib/mqjob/worker.rb
ADDED
@@ -0,0 +1,143 @@
|
|
1
|
+
module Mqjob
|
2
|
+
module Worker
|
3
|
+
SUBSCRIPTION_MODES = [:exclusive, :failover, :shared].freeze
|
4
|
+
|
5
|
+
def initialize(opts)
|
6
|
+
@pool = opts[:pool]
|
7
|
+
@topic = self.class.topic
|
8
|
+
@topic_opts = self.class.topic_opts
|
9
|
+
|
10
|
+
@mq = Plugin.client(@topic_opts[:client])
|
11
|
+
end
|
12
|
+
|
13
|
+
def ack!; :ack end
|
14
|
+
def reject!; :reject; end
|
15
|
+
def requeue!; :requeue; end
|
16
|
+
|
17
|
+
def do_work(cmd, msg)
|
18
|
+
@pool.post do
|
19
|
+
begin
|
20
|
+
wrap_perform = ::Mqjob.hooks&.wrap_perform
|
21
|
+
|
22
|
+
::Mqjob.logger.debug(__method__){'Begin process'}
|
23
|
+
|
24
|
+
if wrap_perform.nil?
|
25
|
+
process_work(cmd, msg)
|
26
|
+
else
|
27
|
+
wrap_perform.call do
|
28
|
+
process_work(cmd, msg)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
::Mqjob.logger.debug(__method__){'Finish process'}
|
33
|
+
rescue => exp
|
34
|
+
::Mqjob.logger.error(__method__){"message process error: #{exp.message}! cmd: #{cmd}, msg: #{msg}"}
|
35
|
+
::Mqjob.logger.error(__method__){exp}
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def run
|
41
|
+
@mq.listen(@topic, self, @topic_opts)
|
42
|
+
end
|
43
|
+
|
44
|
+
def stop
|
45
|
+
@mq.close_listen
|
46
|
+
end
|
47
|
+
|
48
|
+
def perform(msg); end
|
49
|
+
|
50
|
+
private
|
51
|
+
def process_work(cmd, msg)
|
52
|
+
RequestStore.clear! if Object.const_defined?(:RequestStore)
|
53
|
+
|
54
|
+
result = nil
|
55
|
+
begin
|
56
|
+
result = if respond_to?(:perform_full_msg)
|
57
|
+
::Mqjob.logger.info(__method__){"perform_full_msg: #{msg.inspect}"}
|
58
|
+
perform_full_msg(cmd, msg)
|
59
|
+
else
|
60
|
+
::Mqjob.logger.info(__method__){"perform: #{msg.payload.inspect}"}
|
61
|
+
perform(msg.payload)
|
62
|
+
end
|
63
|
+
rescue => exp
|
64
|
+
::Mqjob.logger.error(__method__){exp}
|
65
|
+
result = :error
|
66
|
+
end
|
67
|
+
|
68
|
+
case result
|
69
|
+
when :error, :reject
|
70
|
+
::Mqjob.logger.info(__method__) {"Redeliver messages! Current message id is: #{msg.message_id.inspect}"}
|
71
|
+
msg.nack
|
72
|
+
when :requeue
|
73
|
+
::Mqjob.logger.info(__method__) {"Requeue! message id is: #{msg.message_id.inspect}"}
|
74
|
+
msg.ack
|
75
|
+
self.class.enqueue(msg.payload, in: 10)
|
76
|
+
else
|
77
|
+
::Mqjob.logger.info(__method__) {"Acknowledge message!"}
|
78
|
+
msg.ack
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
def self.included(base)
|
83
|
+
base.extend ClassMethods
|
84
|
+
::Mqjob.regist_class(base) if base.is_a? Class
|
85
|
+
end
|
86
|
+
|
87
|
+
module ClassMethods
|
88
|
+
attr_reader :topic_opts
|
89
|
+
attr_reader :topic
|
90
|
+
|
91
|
+
# client: MQ,
|
92
|
+
# plugin: :pulsar,
|
93
|
+
# prefetch: 1,
|
94
|
+
# subscription_mode: SUBSCRIPTION_MODES, # 不同类型需要不同配置参数,互斥模式下需要指定订阅名
|
95
|
+
# subscription_name
|
96
|
+
# logger: MyLogger
|
97
|
+
# topic_type [:normal, :regex] default normal
|
98
|
+
def from_topic(name, opts={})
|
99
|
+
@topic = name.respond_to?(:call) ? name.call : name
|
100
|
+
@topic_opts = opts
|
101
|
+
|
102
|
+
topic_type = @topic_opts[:topic_type]&.to_sym
|
103
|
+
@topic_opts[:topic_type] = topic_type || :normal
|
104
|
+
|
105
|
+
@topic_opts[:subscription_name] ||= (self.name.split('::') << 'Consumer').join
|
106
|
+
end
|
107
|
+
|
108
|
+
# opts
|
109
|
+
# in publish message in X seconds
|
110
|
+
# at publish message at specific time
|
111
|
+
# init_subscription Boolean 是否先初始化一个订阅
|
112
|
+
# perform_now Boolean 立即执行,通常用于测试环境减少流程
|
113
|
+
def enqueue(msg, opts={})
|
114
|
+
if topic_opts[:topic_type] != :normal
|
115
|
+
::Mqjob.logger.error(__method__){
|
116
|
+
"message enqueue only support topic_type set to normal, but got 「#{topic_opts[:topic_type]}」! After action skipped!"
|
117
|
+
}
|
118
|
+
return false
|
119
|
+
end
|
120
|
+
|
121
|
+
if !opts[:perform_now]
|
122
|
+
@mq ||= Plugin.client(topic_opts[:client])
|
123
|
+
@mq.publish(topic, msg, topic_opts.merge(opts))
|
124
|
+
return true
|
125
|
+
end
|
126
|
+
|
127
|
+
begin
|
128
|
+
worker = self.new({})
|
129
|
+
if worker.respond_to?(:perform)
|
130
|
+
msg = JSON.parse(JSON.dump(msg))
|
131
|
+
::Mqjob.logger.info('perform message now'){msg.inspect}
|
132
|
+
worker.send(:process_work, nil, OpenStruct.new(payload: msg))
|
133
|
+
else
|
134
|
+
::Mqjob.logger.error('perform_now required 「perform」 method, 「perform_full_msg」not supported!')
|
135
|
+
end
|
136
|
+
rescue => exp
|
137
|
+
::Mqjob.logger.error("#{self.name} perform_now") {exp}
|
138
|
+
end
|
139
|
+
true
|
140
|
+
end
|
141
|
+
end
|
142
|
+
end
|
143
|
+
end
|
@@ -0,0 +1,71 @@
|
|
1
|
+
module Mqjob
|
2
|
+
module WorkerGroup
|
3
|
+
def initialize
|
4
|
+
@stoped = false
|
5
|
+
# 统一线程池,防止数据库连接池不够用,推荐设置为10
|
6
|
+
@pool = ::Mqjob::ThreadPool.new(threads)
|
7
|
+
end
|
8
|
+
|
9
|
+
def before_fork
|
10
|
+
::Mqjob.hooks.before_fork&.call
|
11
|
+
end
|
12
|
+
|
13
|
+
def run
|
14
|
+
return if @stoped
|
15
|
+
|
16
|
+
::Mqjob.hooks.after_fork&.call
|
17
|
+
|
18
|
+
workers.each do |worker|
|
19
|
+
worker.run
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def stop
|
24
|
+
workers.each{|wc| wc.stop}
|
25
|
+
|
26
|
+
@stoped = true
|
27
|
+
end
|
28
|
+
|
29
|
+
def reload
|
30
|
+
puts "call #{__method__}"
|
31
|
+
end
|
32
|
+
|
33
|
+
def after_start
|
34
|
+
puts "call #{__method__}"
|
35
|
+
end
|
36
|
+
|
37
|
+
def workers
|
38
|
+
@workers ||= worker_classes.map do |wc|
|
39
|
+
wc.new(pool: @pool)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
# 设置线程数并返回新类型
|
44
|
+
# opts
|
45
|
+
# threads 设置线程池大小
|
46
|
+
def self.configure(opts)
|
47
|
+
thread_size = opts[:threads]
|
48
|
+
raise "threads was required!" if thread_size.to_i.zero?
|
49
|
+
workers = Array(opts[:clazz])
|
50
|
+
raise "clazz was required!" if workers.empty?
|
51
|
+
|
52
|
+
md = Module.new
|
53
|
+
md_name = "WorkerGroup#{md.object_id}".tr('-', '_')
|
54
|
+
|
55
|
+
md.include(self)
|
56
|
+
md.class_eval <<-RUBY, __FILE__, __LINE__+1
|
57
|
+
def threads
|
58
|
+
#{thread_size}
|
59
|
+
end
|
60
|
+
|
61
|
+
def worker_classes
|
62
|
+
#{workers}
|
63
|
+
end
|
64
|
+
|
65
|
+
private :threads, :workers
|
66
|
+
RUBY
|
67
|
+
|
68
|
+
::Mqjob.const_set(md_name, md)
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
data/lib/plugin.rb
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
require 'plugin/base'
|
2
|
+
|
3
|
+
module Plugin
|
4
|
+
extend self
|
5
|
+
|
6
|
+
def client(c, plugin: nil)
|
7
|
+
plugin = init_plugin(plugin)
|
8
|
+
c ||= Mqjob.default_client
|
9
|
+
|
10
|
+
Plugin.const_get(plugin).new(c)
|
11
|
+
end
|
12
|
+
|
13
|
+
def init_plugin(name)
|
14
|
+
name ||= Mqjob.config.plugin
|
15
|
+
|
16
|
+
Mqjob.logger.debug("#{self.name}::#{__method__}"){"select plugin: #{name}"}
|
17
|
+
|
18
|
+
require "plugin/#{name}"
|
19
|
+
|
20
|
+
name.to_s.capitalize
|
21
|
+
end
|
22
|
+
|
23
|
+
private :init_plugin
|
24
|
+
end
|
data/lib/plugin/base.rb
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
module Plugin
|
2
|
+
class Base
|
3
|
+
def initialize(client)
|
4
|
+
@client = client
|
5
|
+
@subscription_mode = ::Mqjob.config.subscription_mode
|
6
|
+
end
|
7
|
+
|
8
|
+
def listen(topic, worker, opts = {}); end
|
9
|
+
|
10
|
+
def publish(topic, msg, opts = {}); end
|
11
|
+
|
12
|
+
def close_listen; end
|
13
|
+
def close_publish; end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,92 @@
|
|
1
|
+
require 'pulsar_sdk'
|
2
|
+
|
3
|
+
module Plugin
|
4
|
+
class Pulsar < Base
|
5
|
+
def listen(topic, worker, opts = {})
|
6
|
+
create_consumer(topic, opts).listen do |cmd, msg|
|
7
|
+
::Mqjob.logger.debug("#{self.class.name}::#{__method__}"){"receive msg: #{msg.payload}"}
|
8
|
+
worker.do_work(cmd, msg)
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
# opts
|
13
|
+
# in publish message in X seconds
|
14
|
+
# at publish message at specific time
|
15
|
+
# init_subscription Boolean
|
16
|
+
def publish(topic, msg, opts = {})
|
17
|
+
create_consumer(topic, opts) if opts[:init_subscription]
|
18
|
+
|
19
|
+
base_cmd = ::Pulsar::Proto::BaseCommand.new(
|
20
|
+
type: ::Pulsar::Proto::BaseCommand::Type::SEND,
|
21
|
+
send: ::Pulsar::Proto::CommandSend.new(
|
22
|
+
num_messages: 1
|
23
|
+
)
|
24
|
+
)
|
25
|
+
|
26
|
+
get_timestamp = lambda {|v| (v.to_f * 1000).floor}
|
27
|
+
|
28
|
+
deliver_at = case
|
29
|
+
when opts[:in]
|
30
|
+
Time.now.localtime + opts[:in].to_f
|
31
|
+
when opts[:at]
|
32
|
+
opts[:at]
|
33
|
+
else
|
34
|
+
Time.now.localtime
|
35
|
+
end
|
36
|
+
|
37
|
+
metadata = ::Pulsar::Proto::MessageMetadata.new(
|
38
|
+
deliver_at_time: get_timestamp.call(deliver_at)
|
39
|
+
)
|
40
|
+
p_msg = ::PulsarSdk::Producer::Message.new(msg, metadata)
|
41
|
+
|
42
|
+
create_producer(topic, opts).execute_async(base_cmd, p_msg)
|
43
|
+
end
|
44
|
+
|
45
|
+
def close_listen
|
46
|
+
@consumer&.close
|
47
|
+
end
|
48
|
+
|
49
|
+
def close_publish
|
50
|
+
@producer&.close
|
51
|
+
end
|
52
|
+
|
53
|
+
private
|
54
|
+
def create_consumer(topic, opts)
|
55
|
+
@consumer ||= begin
|
56
|
+
topic_type = :topic
|
57
|
+
if opts[:topic_type].to_sym == :regex
|
58
|
+
topic_type = :topics_pattern
|
59
|
+
elsif topic.is_a?(Array)
|
60
|
+
topic_type = :topics
|
61
|
+
end
|
62
|
+
|
63
|
+
consumer_opts = ::PulsarSdk::Options::Consumer.new(
|
64
|
+
topic_type => topic,
|
65
|
+
subscription_type: (opts[:subscription_mode] || @subscription_mode).to_s.capitalize.to_sym,
|
66
|
+
subscription_name: opts[:subscription_name],
|
67
|
+
prefetch: opts[:prefetch] || 1,
|
68
|
+
listen_wait: 0.1
|
69
|
+
)
|
70
|
+
|
71
|
+
::Mqjob.logger.debug(__method__){consumer_opts.inspect}
|
72
|
+
|
73
|
+
@client.subscribe(consumer_opts)
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
def create_producer(topic, opts)
|
78
|
+
@producer ||= begin
|
79
|
+
producer_opts = ::PulsarSdk::Options::Producer.new(
|
80
|
+
topic: topic
|
81
|
+
)
|
82
|
+
|
83
|
+
@client.create_producer(producer_opts)
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
# NOTE
|
91
|
+
# ServerEngine会自动循环调用,所以这里使用无阻塞listen即可
|
92
|
+
# 如果这里listen阻塞会导致同组任务无法正常执行
|
data/mqjob.gemspec
ADDED
@@ -0,0 +1,30 @@
|
|
1
|
+
|
2
|
+
lib = File.expand_path("../lib", __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require "mqjob/version"
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "mqjob"
|
8
|
+
spec.version = Mqjob::VERSION
|
9
|
+
spec.authors = 'archfish'
|
10
|
+
spec.email = ["weihailang@gmail.com"]
|
11
|
+
spec.license = 'Apache License 2.0'
|
12
|
+
|
13
|
+
spec.summary = %q{A queue job base on Apache Pulsar}
|
14
|
+
spec.description = %q{A queue job base on Apache Pulsar}
|
15
|
+
spec.homepage = "https://github.com/archfish/mqjob"
|
16
|
+
|
17
|
+
# Specify which files should be added to the gem when it is released.
|
18
|
+
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
19
|
+
spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do
|
20
|
+
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features|examples)/}) }
|
21
|
+
end
|
22
|
+
|
23
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
24
|
+
spec.require_paths = ["lib"]
|
25
|
+
|
26
|
+
spec.add_dependency 'serverengine', '~> 2.2'
|
27
|
+
spec.add_dependency 'concurrent-ruby', '~> 1.1'
|
28
|
+
|
29
|
+
spec.add_development_dependency "bundler", "> 1.17"
|
30
|
+
end
|
metadata
ADDED
@@ -0,0 +1,100 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: mqjob
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.4.6
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- archfish
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2020-05-26 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: serverengine
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '2.2'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '2.2'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: concurrent-ruby
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '1.1'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '1.1'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: bundler
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '1.17'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '1.17'
|
55
|
+
description: A queue job base on Apache Pulsar
|
56
|
+
email:
|
57
|
+
- weihailang@gmail.com
|
58
|
+
executables: []
|
59
|
+
extensions: []
|
60
|
+
extra_rdoc_files: []
|
61
|
+
files:
|
62
|
+
- ".gitignore"
|
63
|
+
- Gemfile
|
64
|
+
- README.md
|
65
|
+
- Rakefile
|
66
|
+
- bin/console
|
67
|
+
- bin/setup
|
68
|
+
- lib/mqjob.rb
|
69
|
+
- lib/mqjob/thread_pool.rb
|
70
|
+
- lib/mqjob/version.rb
|
71
|
+
- lib/mqjob/worker.rb
|
72
|
+
- lib/mqjob/worker_group.rb
|
73
|
+
- lib/plugin.rb
|
74
|
+
- lib/plugin/base.rb
|
75
|
+
- lib/plugin/pulsar.rb
|
76
|
+
- mqjob.gemspec
|
77
|
+
homepage: https://github.com/archfish/mqjob
|
78
|
+
licenses:
|
79
|
+
- Apache License 2.0
|
80
|
+
metadata: {}
|
81
|
+
post_install_message:
|
82
|
+
rdoc_options: []
|
83
|
+
require_paths:
|
84
|
+
- lib
|
85
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
91
|
+
requirements:
|
92
|
+
- - ">="
|
93
|
+
- !ruby/object:Gem::Version
|
94
|
+
version: '0'
|
95
|
+
requirements: []
|
96
|
+
rubygems_version: 3.0.8
|
97
|
+
signing_key:
|
98
|
+
specification_version: 4
|
99
|
+
summary: A queue job base on Apache Pulsar
|
100
|
+
test_files: []
|