qpush 0.1.8 → 0.1.10
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.byebug_history +0 -242
- data/README.md +4 -2
- data/jobs/fail_job.rb +1 -1
- data/jobs/test_job.rb +2 -2
- data/lib/qpush/base.rb +0 -1
- data/lib/qpush/base/job.rb +1 -0
- data/lib/qpush/base/redis.rb +35 -4
- data/lib/qpush/client.rb +23 -4
- data/lib/qpush/server.rb +2 -3
- data/lib/qpush/server/apis/setup.rb +3 -4
- data/lib/qpush/server/apis/success.rb +1 -3
- data/lib/qpush/server/config.rb +22 -28
- data/lib/qpush/server/jobs.rb +18 -10
- data/lib/qpush/server/launcher.rb +11 -1
- data/lib/qpush/server/logger.rb +0 -2
- data/lib/qpush/server/manager.rb +1 -26
- data/lib/qpush/server/worker.rb +73 -54
- data/lib/qpush/version.rb +1 -1
- data/lib/qpush/web.rb +0 -1
- data/lib/qpush/web/config.rb +1 -4
- metadata +2 -8
- data/dump.rdb +0 -0
- data/lib/qpush/client/config.rb +0 -13
- data/lib/qpush/client/job.rb +0 -12
- data/lib/qpush/client/redis.rb +0 -16
- data/lib/qpush/server/redis.rb +0 -34
- data/lib/qpush/web/redis.rb +0 -48
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ca96ec0ea37c952b0dc88b82129fae5f60ab604a
|
4
|
+
data.tar.gz: f6d0ef10bf31c71e9a8a67d91830fb007500a8dd
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 453061fe039e72dea482d0ba003fab94de6a20ffb18f8076ea0eb68e36fc72c3182018bafb5ee4e2f0c616c9c8cee32be3fa4765103c8b368fe69c269ec0587e
|
7
|
+
data.tar.gz: 32bb65924fe5b3189d5f4f7d24d46fc5cda3cd3812458e3f54d5eacc5ccd2fb7cbe147005319468cb1d87565c8fd8eb2d6fd0e06a024444cd44e0ad9eb8417c9
|
data/.byebug_history
CHANGED
@@ -1,243 +1 @@
|
|
1
1
|
c
|
2
|
-
job.valid?
|
3
|
-
job.setup
|
4
|
-
job
|
5
|
-
c
|
6
|
-
json
|
7
|
-
c
|
8
|
-
self.config
|
9
|
-
config
|
10
|
-
Server.config
|
11
|
-
c
|
12
|
-
Server.config
|
13
|
-
c
|
14
|
-
QPush::Server.config
|
15
|
-
c
|
16
|
-
QPush::Server.config
|
17
|
-
c
|
18
|
-
job.perform
|
19
|
-
job = Job.new(JSON.parse(json))
|
20
|
-
json
|
21
|
-
c
|
22
|
-
delays.count
|
23
|
-
delays.any?
|
24
|
-
delays
|
25
|
-
@conn
|
26
|
-
c
|
27
|
-
valid?
|
28
|
-
c
|
29
|
-
configs
|
30
|
-
c
|
31
|
-
configs
|
32
|
-
c
|
33
|
-
valid?
|
34
|
-
config
|
35
|
-
c
|
36
|
-
valid?
|
37
|
-
@configs
|
38
|
-
configs
|
39
|
-
c
|
40
|
-
configs ? true : false
|
41
|
-
configs
|
42
|
-
c
|
43
|
-
valid?
|
44
|
-
quit
|
45
|
-
job.klass
|
46
|
-
job.errors
|
47
|
-
job.valid?
|
48
|
-
c
|
49
|
-
jobs
|
50
|
-
c
|
51
|
-
request.path_info
|
52
|
-
c
|
53
|
-
hash
|
54
|
-
c
|
55
|
-
@jobs.compact[@start, @count]
|
56
|
-
@jobs.compact
|
57
|
-
@jobs.compact!
|
58
|
-
@count
|
59
|
-
@start
|
60
|
-
@jobs
|
61
|
-
c
|
62
|
-
@count
|
63
|
-
@start
|
64
|
-
@jobs
|
65
|
-
c
|
66
|
-
Delay.call(@job, :delay)
|
67
|
-
c
|
68
|
-
Delay.call(@job)
|
69
|
-
c
|
70
|
-
@stats.each_key { |key| puts key }
|
71
|
-
@stats.each_key { |key| @stats[key].to_i }
|
72
|
-
@stats
|
73
|
-
c
|
74
|
-
@stats
|
75
|
-
c
|
76
|
-
quit
|
77
|
-
Perform.call(@job)
|
78
|
-
c
|
79
|
-
Perform.call(@job)
|
80
|
-
c
|
81
|
-
@job
|
82
|
-
invalid_job && return unless @job.valid?
|
83
|
-
c
|
84
|
-
job
|
85
|
-
c
|
86
|
-
@fails = @fails.compact[@start, @count]
|
87
|
-
@fails
|
88
|
-
@Fails
|
89
|
-
@fails.compact!
|
90
|
-
@fails.compact![@start..@count]
|
91
|
-
@fails.compact![@start, @count]
|
92
|
-
@fails.any?
|
93
|
-
@count
|
94
|
-
@start
|
95
|
-
@fails
|
96
|
-
c
|
97
|
-
@fails
|
98
|
-
c
|
99
|
-
@fails
|
100
|
-
c
|
101
|
-
percent
|
102
|
-
quit
|
103
|
-
y
|
104
|
-
q
|
105
|
-
(100.00 - ((stats[:failed].to_f / stats[:performed].to_f) * 100.00)).round(2)
|
106
|
-
100.00 - ((stats[:failed].to_f / stats[:performed].to_f) * 100.00)
|
107
|
-
stats[:failed].to_f / stats[:performed].to_f
|
108
|
-
stats[:failed].to_f
|
109
|
-
stats[:failed].to_i / stats[:performed].to_i
|
110
|
-
(stats[:failed] / stats[:performed])
|
111
|
-
stats[:performed]
|
112
|
-
stats[:failed]
|
113
|
-
percent
|
114
|
-
c
|
115
|
-
conn.lrange("#{QPush.config.perform_namespace}:last_10", 0, 10)
|
116
|
-
c
|
117
|
-
json
|
118
|
-
c
|
119
|
-
conn.get("#{namespace}:#{s}")
|
120
|
-
stats
|
121
|
-
c
|
122
|
-
nil.to_i
|
123
|
-
stats
|
124
|
-
quit
|
125
|
-
Dir[Dir.pwd + "/lib/qpush/jobs/*.rb"]
|
126
|
-
Dir[Dir.pwd + "/lib/jobs/*.rb"]
|
127
|
-
Dir.pwd
|
128
|
-
File.dirname(__FILE__)
|
129
|
-
Dir["#{File.dirname(__FILE__)}/qpush/jobs/*.rb"]
|
130
|
-
Dir[File.join(QPush.config.jobs_path, "*.rb")]
|
131
|
-
$LOAD_PATH.unshift(QPush.config.jobs_path)
|
132
|
-
$LOAD_PATH
|
133
|
-
File.dirname(__FILE__)
|
134
|
-
Dir[File.dirname(__FILE__) + "#{QPush.config.jobs_path}/*.rb"]
|
135
|
-
QPush.config.jobs_path
|
136
|
-
Dir["#{QPush.config.jobs_path}/*.rb"]
|
137
|
-
quit
|
138
|
-
puts e
|
139
|
-
puts s
|
140
|
-
c
|
141
|
-
puts e
|
142
|
-
puts s
|
143
|
-
s
|
144
|
-
jobs.compact!
|
145
|
-
jobs.compact![s, e]
|
146
|
-
jobs
|
147
|
-
quit
|
148
|
-
jobs
|
149
|
-
c
|
150
|
-
jobs
|
151
|
-
quit
|
152
|
-
job_hash
|
153
|
-
quit
|
154
|
-
delayed_jobs
|
155
|
-
delay_jobs
|
156
|
-
c
|
157
|
-
conn.zrange(QPush.config.delay_namespace, 0, -1, with_scores: true)
|
158
|
-
delayed_jobs
|
159
|
-
quit
|
160
|
-
conn.get("#{@namespace}:#{stat}")
|
161
|
-
"#{@namespace}:#{stat}"
|
162
|
-
c
|
163
|
-
name
|
164
|
-
c
|
165
|
-
self.name
|
166
|
-
quit
|
167
|
-
job
|
168
|
-
delays.each { |job| PerformJob.call(Job.new(job)) }
|
169
|
-
cccccccccccc
|
170
|
-
elcccccccccccccccccccc
|
171
|
-
delaysdelays.each {|job| PerformJob.call(Job.new(job))}
|
172
|
-
quit
|
173
|
-
Server.connection_pool
|
174
|
-
Qued::QPush.redis
|
175
|
-
QPush.redis
|
176
|
-
c
|
177
|
-
QueueJob.new(job).call
|
178
|
-
job.valid? && job.queueable?
|
179
|
-
job.valid?
|
180
|
-
return false unless job.valid? && job.queueable?
|
181
|
-
job
|
182
|
-
quit
|
183
|
-
@start_at < Time.now.to_i && !@cron.empty?
|
184
|
-
@start_at > Time.now.to_i && @cron.empty?
|
185
|
-
@start_at < Time.now.to_i && @cron.empty?
|
186
|
-
QueueJob.new(self).call
|
187
|
-
quit
|
188
|
-
job.valid?
|
189
|
-
job.save
|
190
|
-
test
|
191
|
-
quit
|
192
|
-
job.save
|
193
|
-
job
|
194
|
-
job = Job.new(options)
|
195
|
-
options
|
196
|
-
quit
|
197
|
-
cron?
|
198
|
-
delay?
|
199
|
-
queue?
|
200
|
-
quit
|
201
|
-
c
|
202
|
-
job_type.call
|
203
|
-
quit
|
204
|
-
job_type.call
|
205
|
-
quit
|
206
|
-
performer
|
207
|
-
performer = retrieve_job_type
|
208
|
-
quit
|
209
|
-
self
|
210
|
-
performer.call
|
211
|
-
performer
|
212
|
-
performer = retrieve_job_type
|
213
|
-
performer.call
|
214
|
-
performer
|
215
|
-
retrieve_job_type
|
216
|
-
performer
|
217
|
-
c
|
218
|
-
quit
|
219
|
-
performer.call
|
220
|
-
performer
|
221
|
-
performer = retrieve_job_type
|
222
|
-
valid?
|
223
|
-
c
|
224
|
-
@job.queue?
|
225
|
-
@job.delay?
|
226
|
-
@job.cron?
|
227
|
-
@job
|
228
|
-
job_success
|
229
|
-
measure_run_time { @job.run }
|
230
|
-
@job.run
|
231
|
-
c
|
232
|
-
quit
|
233
|
-
log_success
|
234
|
-
job_success && log_success
|
235
|
-
c
|
236
|
-
@job.run
|
237
|
-
@job = Job.new(JSON.parse(json))
|
238
|
-
c
|
239
|
-
@job = Job.new(JSON.parse(json))
|
240
|
-
json
|
241
|
-
quit
|
242
|
-
json
|
243
|
-
son
|
data/README.md
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
|
1
|
+
#<center><img src="https://s21.postimg.org/85ce7g93b/qpush.png" alt="Drawing" width="250"/></center>
|
2
2
|
[![Code Climate](https://codeclimate.com/github/nsweeting/qpush/badges/gpa.svg)](https://codeclimate.com/github/nsweeting/qpush)
|
3
3
|
|
4
4
|
Fast and simple job queue microservice for Ruby. **Please consider it under development at the moment.**
|
@@ -77,12 +77,14 @@ At a minimum, we must provide the job with a 'klass'. There are many more option
|
|
77
77
|
|
78
78
|
#### Building Jobs
|
79
79
|
|
80
|
-
Jobs are simply
|
80
|
+
Jobs are simply ruby objects that include the QPush::Job module and contain a 'call' method. If you provide a hash for the 'args' of the job, the job will be initialized with them. Below is an example of a simple mailing job utilizing the 'mail' gem.
|
81
81
|
|
82
82
|
```ruby
|
83
83
|
require 'mail'
|
84
84
|
|
85
85
|
class MailJob
|
86
|
+
include QPush::Job
|
87
|
+
|
86
88
|
def initialize(options = {})
|
87
89
|
@mail = Mail.new(options)
|
88
90
|
end
|
data/jobs/fail_job.rb
CHANGED
data/jobs/test_job.rb
CHANGED
data/lib/qpush/base.rb
CHANGED
data/lib/qpush/base/job.rb
CHANGED
data/lib/qpush/base/redis.rb
CHANGED
@@ -1,11 +1,42 @@
|
|
1
1
|
module QPush
|
2
2
|
module Base
|
3
3
|
KEY = 'qpush:v1'.freeze
|
4
|
+
SUB_KEYS = [:delay,
|
5
|
+
:queue,
|
6
|
+
:perform,
|
7
|
+
:stats,
|
8
|
+
:heart,
|
9
|
+
:crons,
|
10
|
+
:history,
|
11
|
+
:morgue].freeze
|
4
12
|
|
5
|
-
|
6
|
-
def self.
|
7
|
-
|
8
|
-
|
13
|
+
module RedisHelper
|
14
|
+
def self.included(base)
|
15
|
+
base.extend(ClassMethods)
|
16
|
+
end
|
17
|
+
|
18
|
+
module ClassMethods
|
19
|
+
def redis
|
20
|
+
redis_pool.with do |conn|
|
21
|
+
yield conn
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def redis_pool
|
26
|
+
@redis_pool ||= build_pool(config.redis_pool, config.redis_url)
|
27
|
+
end
|
28
|
+
|
29
|
+
def build_pool(pool, url)
|
30
|
+
ConnectionPool.new(size: pool) do
|
31
|
+
Redis.new(url: url)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def build_keys(namespace, priorities)
|
36
|
+
name = "#{QPush::Base::KEY}:#{namespace}"
|
37
|
+
keys = Hash[QPush::Base::SUB_KEYS.collect { |key| [key, "#{name}:#{key}"] }]
|
38
|
+
keys[:perform_list] = (1..priorities).collect { |num| "#{keys[:perform]}:#{num}" }
|
39
|
+
keys
|
9
40
|
end
|
10
41
|
end
|
11
42
|
end
|
data/lib/qpush/client.rb
CHANGED
@@ -1,7 +1,26 @@
|
|
1
1
|
# QPush Base
|
2
2
|
require 'qpush/base'
|
3
3
|
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
4
|
+
module QPush
|
5
|
+
module Client
|
6
|
+
include QPush::Base::ConfigHelper
|
7
|
+
include QPush::Base::RedisHelper
|
8
|
+
|
9
|
+
class << self
|
10
|
+
def config
|
11
|
+
@config ||= Config.new
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
class Config < QPush::Base::Config; end
|
16
|
+
|
17
|
+
class Job < QPush::Base::Job
|
18
|
+
def queue
|
19
|
+
Client.redis do |conn|
|
20
|
+
conn.hincrby("#{QPush::Base::KEY}:#{@namespace}:stats", 'queued', 1)
|
21
|
+
conn.lpush("#{QPush::Base::KEY}:#{@namespace}:queue", to_json)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
data/lib/qpush/server.rb
CHANGED
@@ -2,13 +2,14 @@
|
|
2
2
|
require 'sequel'
|
3
3
|
require 'object_validator'
|
4
4
|
require 'parse-cron'
|
5
|
-
require '
|
5
|
+
require 'logger'
|
6
6
|
|
7
7
|
# QPush Base
|
8
8
|
require 'qpush/base'
|
9
9
|
|
10
10
|
# Qpush Server Base
|
11
11
|
require 'qpush/server/apis'
|
12
|
+
require 'qpush/server/worker'
|
12
13
|
require 'qpush/server/config'
|
13
14
|
require 'qpush/server/database'
|
14
15
|
require 'qpush/server/delay'
|
@@ -21,8 +22,6 @@ require 'qpush/server/logger'
|
|
21
22
|
require 'qpush/server/manager'
|
22
23
|
require 'qpush/server/perform'
|
23
24
|
require 'qpush/server/queue'
|
24
|
-
require 'qpush/server/redis'
|
25
|
-
require 'qpush/server/worker'
|
26
25
|
|
27
26
|
# QPush Server Apis
|
28
27
|
require 'qpush/server/apis/delay'
|
@@ -3,15 +3,14 @@ module QPush
|
|
3
3
|
module Apis
|
4
4
|
class Setup < Base
|
5
5
|
def call
|
6
|
-
|
7
|
-
setup_job
|
6
|
+
@job.valid? ? setup_job : invalid_job
|
8
7
|
end
|
9
8
|
|
10
9
|
private
|
11
10
|
|
12
11
|
def setup_job
|
13
|
-
|
14
|
-
|
12
|
+
@job.perform if @job.perform_job?
|
13
|
+
@job.delay if @job.delay_job?
|
15
14
|
end
|
16
15
|
|
17
16
|
def invalid_job
|
data/lib/qpush/server/config.rb
CHANGED
@@ -1,43 +1,19 @@
|
|
1
1
|
module QPush
|
2
2
|
module Server
|
3
3
|
include QPush::Base::ConfigHelper
|
4
|
+
include QPush::Base::RedisHelper
|
4
5
|
|
5
6
|
class << self
|
6
|
-
attr_accessor :
|
7
|
+
attr_accessor :keys
|
7
8
|
|
8
9
|
def config
|
9
10
|
@config ||= Config.new
|
10
11
|
end
|
11
|
-
|
12
|
-
def build_worker
|
13
|
-
worker = WorkerConfig.new
|
14
|
-
yield worker
|
15
|
-
worker
|
16
|
-
end
|
17
|
-
end
|
18
|
-
|
19
|
-
class WorkerConfig
|
20
|
-
DEFAULTS = {
|
21
|
-
namespace: 'default',
|
22
|
-
priorities: 5,
|
23
|
-
queue_threads: 2,
|
24
|
-
perform_threads: 2,
|
25
|
-
delay_threads: 1 }.freeze
|
26
|
-
|
27
|
-
attr_accessor :perform_threads, :queue_threads, :delay_threads,
|
28
|
-
:namespace, :priorities
|
29
|
-
|
30
|
-
def initialize(options = {})
|
31
|
-
options = DEFAULTS.merge(options)
|
32
|
-
options.each { |key, value| send("#{key}=", value) }
|
33
|
-
end
|
34
|
-
|
35
|
-
def for_keys
|
36
|
-
{ namespace: @namespace, priorities: @priorities }
|
37
|
-
end
|
38
12
|
end
|
39
13
|
|
40
14
|
class Config < QPush::Base::Config
|
15
|
+
include ObjectValidator::Validate
|
16
|
+
|
41
17
|
SERVER_DEFAULTS = {
|
42
18
|
database_url: ENV['DATABASE_URL'],
|
43
19
|
database_pool: 10,
|
@@ -50,6 +26,24 @@ module QPush
|
|
50
26
|
super
|
51
27
|
SERVER_DEFAULTS.each { |key, value| send("#{key}=", value) }
|
52
28
|
end
|
29
|
+
|
30
|
+
def validate!
|
31
|
+
return if valid?
|
32
|
+
fail ServerError, errors.full_messages.join(' ')
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
class ConfigValidator
|
37
|
+
include ObjectValidator::Validator
|
38
|
+
|
39
|
+
validates :redis, with: { proc: proc { Server.redis { |c| c.ping && c.quit } },
|
40
|
+
msg: 'could not be connected with' }
|
41
|
+
validates :workers, with: { proc: proc { |c| c.workers.is_a?(Array) && c.workers.count > 0 },
|
42
|
+
msg: 'is not a valid Array of WorkerConfigs' }
|
43
|
+
validates :configs, with: { proc: proc { |c| c.workers.all? { |w| w.is_a?(WorkerConfig) } },
|
44
|
+
msg: 'are not valid WorkerConfig objects' }
|
45
|
+
validates :jobs_path, with: { proc: proc { |c| Dir.exist?(Dir.pwd + c.jobs_path) },
|
46
|
+
msg: 'is not a valid directory' }
|
53
47
|
end
|
54
48
|
end
|
55
49
|
end
|
data/lib/qpush/server/jobs.rb
CHANGED
@@ -1,14 +1,22 @@
|
|
1
1
|
module QPush
|
2
|
+
module Job
|
3
|
+
class << self
|
4
|
+
def included(base)
|
5
|
+
_register_job(base)
|
6
|
+
Server.log.info("* Job loaded: #{base.name}")
|
7
|
+
end
|
8
|
+
|
9
|
+
def _register_job(base)
|
10
|
+
Server.jobs << base.name
|
11
|
+
Server.redis { |c| c.sadd("#{QPush::Base::KEY}:jobs", base.name) }
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
2
16
|
module Server
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
_register_job(base)
|
7
|
-
end
|
8
|
-
|
9
|
-
def _register_job(base)
|
10
|
-
Server.redis { |c| c.sadd("#{QPush::Base::KEY}:jobs", base.name) }
|
11
|
-
end
|
17
|
+
class << self
|
18
|
+
def jobs
|
19
|
+
@jobs ||= []
|
12
20
|
end
|
13
21
|
end
|
14
22
|
|
@@ -69,7 +77,7 @@ module QPush
|
|
69
77
|
include ObjectValidator::Validator
|
70
78
|
|
71
79
|
validates :klass,
|
72
|
-
with: { proc: proc { |j|
|
80
|
+
with: { proc: proc { |j| Server.jobs.include?(j.klass) },
|
73
81
|
msg: 'has not been defined' }
|
74
82
|
validates :cron,
|
75
83
|
with: { proc: proc { |j| j.cron.empty? ? true : CronParser.new(j.cron) },
|
@@ -7,11 +7,12 @@ module QPush
|
|
7
7
|
@argv = argv
|
8
8
|
end
|
9
9
|
|
10
|
-
#
|
10
|
+
# Provides the main entrypoint for starting a QPush server.
|
11
11
|
#
|
12
12
|
def start
|
13
13
|
start_message
|
14
14
|
setup_options
|
15
|
+
validate!
|
15
16
|
setup_jobs
|
16
17
|
boot_manager
|
17
18
|
end
|
@@ -39,12 +40,21 @@ module QPush
|
|
39
40
|
parser.parse!(@argv)
|
40
41
|
end
|
41
42
|
|
43
|
+
# Validates our server and worker configuration.
|
44
|
+
#
|
45
|
+
def validate!
|
46
|
+
Server.config.validate!
|
47
|
+
Server.config.workers.each { |w| w.validate! }
|
48
|
+
end
|
49
|
+
|
42
50
|
# Requires all base jobs as well as user jobs.
|
43
51
|
#
|
44
52
|
def setup_jobs
|
45
53
|
JobLoader.call
|
46
54
|
end
|
47
55
|
|
56
|
+
# Boots our manager
|
57
|
+
#
|
48
58
|
def boot_manager
|
49
59
|
manager = Manager.new(Server.config.workers)
|
50
60
|
manager.start
|
data/lib/qpush/server/logger.rb
CHANGED
data/lib/qpush/server/manager.rb
CHANGED
@@ -4,8 +4,6 @@ module QPush
|
|
4
4
|
# of them to start and shutdown.
|
5
5
|
#
|
6
6
|
class Manager
|
7
|
-
include ObjectValidator::Validate
|
8
|
-
|
9
7
|
attr_accessor :configs
|
10
8
|
attr_reader :forks
|
11
9
|
|
@@ -21,7 +19,6 @@ module QPush
|
|
21
19
|
# sleep so that our Workers can do their thing.
|
22
20
|
#
|
23
21
|
def start
|
24
|
-
validate!
|
25
22
|
start_messages
|
26
23
|
flush_spaces
|
27
24
|
create_workers
|
@@ -55,34 +52,12 @@ module QPush
|
|
55
52
|
Server.log.info("* Worker count: #{@configs.count}")
|
56
53
|
end
|
57
54
|
|
58
|
-
# Validates our data before starting our Workers. Also instantiates our
|
59
|
-
# connection pool by pinging Redis.
|
60
|
-
#
|
61
|
-
def validate!
|
62
|
-
return if valid?
|
63
|
-
fail ServerError, errors.full_messages.join(' ')
|
64
|
-
end
|
65
|
-
|
66
55
|
# Removes the list of namespaces used by our server from Redis. This
|
67
56
|
# prepares it for the new list that will be created by our workers.
|
68
57
|
#
|
69
58
|
def flush_spaces
|
70
|
-
Server.redis { |c| c.del(QPush::Base::KEY
|
59
|
+
Server.redis { |c| c.del("#{QPush::Base::KEY}:namespaces") }
|
71
60
|
end
|
72
61
|
end
|
73
|
-
|
74
|
-
# The ManagerValidator ensures the data for our manager is valid before
|
75
|
-
# attempting to start it.
|
76
|
-
#
|
77
|
-
class ManagerValidator
|
78
|
-
include ObjectValidator::Validator
|
79
|
-
|
80
|
-
validates :redis, with: { proc: proc { Server.redis { |c| c.ping && c.quit } },
|
81
|
-
msg: 'could not be connected with' }
|
82
|
-
validates :configs, with: { proc: proc { |m| m.configs.count > 0 },
|
83
|
-
msg: 'were not defined' }
|
84
|
-
validates :configs, with: { proc: proc { |m| m.configs.each { |c| c.is_a?(WorkerConfig) } },
|
85
|
-
msg: 'are not valid WorkerConfig objects' }
|
86
|
-
end
|
87
62
|
end
|
88
63
|
end
|
data/lib/qpush/server/worker.rb
CHANGED
@@ -1,20 +1,26 @@
|
|
1
|
-
|
2
1
|
module QPush
|
3
2
|
module Server
|
4
|
-
|
5
|
-
|
3
|
+
class << self
|
4
|
+
attr_accessor :worker
|
5
|
+
|
6
|
+
# A convenience method used to create new WorkerConfig objects for use
|
7
|
+
# in our server configuration.
|
8
|
+
#
|
9
|
+
def build_worker
|
10
|
+
worker = WorkerConfig.new
|
11
|
+
yield worker
|
12
|
+
worker
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
# The Worker manages our actions - Queue, Delay, Perform and Heartbeat.
|
17
|
+
# Each of these actions is alloted a number of threads. Each action
|
6
18
|
# object maintains control of these threads through the aptly named start
|
7
19
|
# and shutdown methods.
|
8
20
|
#
|
9
21
|
class Worker
|
10
|
-
extend Forwardable
|
11
|
-
include ObjectValidator::Validate
|
12
|
-
|
13
22
|
attr_reader :config, :pid, :id
|
14
23
|
|
15
|
-
def_delegators :@config, :perform_threads, :delay_threads, :queue_threads,
|
16
|
-
:priorities, :base_threads, :namespace
|
17
|
-
|
18
24
|
def initialize(id, config)
|
19
25
|
@id = id
|
20
26
|
@pid = Process.pid
|
@@ -27,11 +33,10 @@ module QPush
|
|
27
33
|
# Starts our new worker.
|
28
34
|
#
|
29
35
|
def start
|
30
|
-
validate!
|
31
36
|
assign_globals
|
32
37
|
register_space
|
33
38
|
start_message
|
34
|
-
|
39
|
+
build_actions
|
35
40
|
start_threads
|
36
41
|
end
|
37
42
|
|
@@ -45,42 +50,56 @@ module QPush
|
|
45
50
|
|
46
51
|
private
|
47
52
|
|
48
|
-
#
|
49
|
-
# our Queue and Retry objects. We then start them and join them to the
|
50
|
-
# main process.
|
53
|
+
# Assign the globals that are required for our worker to function.
|
51
54
|
#
|
52
|
-
def
|
53
|
-
@
|
54
|
-
|
55
|
+
def assign_globals
|
56
|
+
Server.keys = Server.build_keys(@config.namespace, @config.priorities)
|
57
|
+
Server.worker = self
|
58
|
+
end
|
59
|
+
|
60
|
+
# Registers our workers namespace on Redis
|
61
|
+
#
|
62
|
+
def register_space
|
63
|
+
Server.redis do |c|
|
64
|
+
c.sadd("#{QPush::Base::KEY}:namespaces", @config.namespace)
|
55
65
|
end
|
56
|
-
|
66
|
+
end
|
67
|
+
|
68
|
+
# Information about the start process
|
69
|
+
#
|
70
|
+
def start_message
|
71
|
+
Server.log.info("* Worker #{@id} started | pid: #{@pid} | namespace: #{@config.namespace}")
|
57
72
|
end
|
58
73
|
|
59
74
|
# Instantiates our Queue, Perform, Delay and Heartbeat objects based on
|
60
|
-
# the number of threads specified for each
|
75
|
+
# the number of threads specified for each action type. We store these
|
61
76
|
# objects as an array in @actions.
|
62
77
|
#
|
63
|
-
def
|
64
|
-
|
65
|
-
|
66
|
-
@actions <<
|
78
|
+
def build_actions
|
79
|
+
base_actions.each do |action|
|
80
|
+
action[:count].times do
|
81
|
+
@actions << action[:klass].new
|
67
82
|
end
|
68
83
|
end
|
69
84
|
end
|
70
85
|
|
71
|
-
def
|
86
|
+
def base_actions
|
72
87
|
[
|
73
|
-
{ klass: Perform, count: perform_threads },
|
74
|
-
{ klass: Queue, count: queue_threads },
|
75
|
-
{ klass: Delay, count: delay_threads },
|
88
|
+
{ klass: Perform, count: @config.perform_threads },
|
89
|
+
{ klass: Queue, count: @config.queue_threads },
|
90
|
+
{ klass: Delay, count: @config.delay_threads },
|
76
91
|
{ klass: Heartbeat, count: 1 }
|
77
92
|
]
|
78
93
|
end
|
79
94
|
|
80
|
-
#
|
95
|
+
# Creates threads for each of the action objects, We then start them and
|
96
|
+
# join them to the main process.
|
81
97
|
#
|
82
|
-
def
|
83
|
-
|
98
|
+
def start_threads
|
99
|
+
@actions.each do |action|
|
100
|
+
@threads << Thread.new { action.start }
|
101
|
+
end
|
102
|
+
@threads.map(&:join)
|
84
103
|
end
|
85
104
|
|
86
105
|
# Information about the shutdown process
|
@@ -88,40 +107,40 @@ module QPush
|
|
88
107
|
def shutdown_message
|
89
108
|
Server.log.info("* Worker #{@id} shutdown | pid: #{@pid}")
|
90
109
|
end
|
110
|
+
end
|
91
111
|
|
92
|
-
|
93
|
-
|
94
|
-
def validate!
|
95
|
-
return if valid?
|
96
|
-
fail ServerError, errors.full_messages.join(' ')
|
97
|
-
end
|
112
|
+
class WorkerConfig
|
113
|
+
include ObjectValidator::Validate
|
98
114
|
|
99
|
-
|
100
|
-
|
101
|
-
|
115
|
+
DEFAULTS = {
|
116
|
+
namespace: 'default',
|
117
|
+
priorities: 5,
|
118
|
+
queue_threads: 2,
|
119
|
+
perform_threads: 2,
|
120
|
+
delay_threads: 1 }.freeze
|
121
|
+
|
122
|
+
attr_accessor :perform_threads, :queue_threads, :delay_threads,
|
123
|
+
:namespace, :priorities
|
124
|
+
|
125
|
+
def initialize(options = {})
|
126
|
+
options = DEFAULTS.merge(options)
|
127
|
+
options.each { |key, value| send("#{key}=", value) }
|
102
128
|
end
|
103
129
|
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
Server.redis do |c|
|
108
|
-
c.sadd(QPush::Base::KEY + ':namespaces', namespace)
|
109
|
-
end
|
130
|
+
def validate!
|
131
|
+
return if valid?
|
132
|
+
fail ServerError, errors.full_messages.join(' ')
|
110
133
|
end
|
111
134
|
end
|
112
135
|
|
113
|
-
|
114
|
-
# valid before attempting to use it.
|
115
|
-
#
|
116
|
-
class WorkerValidator
|
136
|
+
class WorkerConfigValidator
|
117
137
|
include ObjectValidator::Validator
|
118
138
|
|
119
|
-
validates :config, type: QPush::Server::WorkerConfig
|
120
|
-
validates :perform_threads, type: Integer, greater_than: 0
|
121
|
-
validates :queue_threads, type: Integer, greater_than: 0
|
122
|
-
validates :delay_threads, type: Integer, greater_than: 0
|
123
139
|
validates :namespace, type: String
|
124
|
-
validates :priorities,
|
140
|
+
validates :priorities, greater_than: 4
|
141
|
+
validates :queue_threads, greater_than: 0
|
142
|
+
validates :perform_threads, greater_than: 0
|
143
|
+
validates :delay_threads, greater_than: 0
|
125
144
|
end
|
126
145
|
end
|
127
146
|
end
|
data/lib/qpush/version.rb
CHANGED
data/lib/qpush/web.rb
CHANGED
data/lib/qpush/web/config.rb
CHANGED
@@ -1,15 +1,12 @@
|
|
1
1
|
module QPush
|
2
2
|
module Web
|
3
3
|
include QPush::Base::ConfigHelper
|
4
|
+
include QPush::Base::RedisHelper
|
4
5
|
|
5
6
|
class << self
|
6
7
|
def config
|
7
8
|
@config ||= Config.new
|
8
9
|
end
|
9
|
-
|
10
|
-
def keys
|
11
|
-
@keys ||= QPush::Web::RedisKeys.new
|
12
|
-
end
|
13
10
|
end
|
14
11
|
|
15
12
|
class Config < QPush::Base::Config; end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: qpush
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.10
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Nicholas Sweeting
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-08-
|
11
|
+
date: 2016-08-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: redis
|
@@ -200,7 +200,6 @@ files:
|
|
200
200
|
- bin/qpush-server
|
201
201
|
- bin/qpush-web
|
202
202
|
- bin/setup
|
203
|
-
- dump.rdb
|
204
203
|
- jobs/fail_job.rb
|
205
204
|
- jobs/test_job.rb
|
206
205
|
- lib/qpush.rb
|
@@ -209,9 +208,6 @@ files:
|
|
209
208
|
- lib/qpush/base/job.rb
|
210
209
|
- lib/qpush/base/redis.rb
|
211
210
|
- lib/qpush/client.rb
|
212
|
-
- lib/qpush/client/config.rb
|
213
|
-
- lib/qpush/client/job.rb
|
214
|
-
- lib/qpush/client/redis.rb
|
215
211
|
- lib/qpush/jobs/queue_delayed.rb
|
216
212
|
- lib/qpush/server.rb
|
217
213
|
- lib/qpush/server/apis.rb
|
@@ -236,7 +232,6 @@ files:
|
|
236
232
|
- lib/qpush/server/manager.rb
|
237
233
|
- lib/qpush/server/perform.rb
|
238
234
|
- lib/qpush/server/queue.rb
|
239
|
-
- lib/qpush/server/redis.rb
|
240
235
|
- lib/qpush/server/worker.rb
|
241
236
|
- lib/qpush/version.rb
|
242
237
|
- lib/qpush/web.rb
|
@@ -267,7 +262,6 @@ files:
|
|
267
262
|
- lib/qpush/web/public/js/app.js
|
268
263
|
- lib/qpush/web/public/js/shims.js
|
269
264
|
- lib/qpush/web/public/tsconfig.json
|
270
|
-
- lib/qpush/web/redis.rb
|
271
265
|
- lib/qpush/web/server.rb
|
272
266
|
- lib/qpush/web/server.ru
|
273
267
|
- qpush.gemspec
|
data/dump.rdb
DELETED
Binary file
|
data/lib/qpush/client/config.rb
DELETED
data/lib/qpush/client/job.rb
DELETED
@@ -1,12 +0,0 @@
|
|
1
|
-
module QPush
|
2
|
-
module Client
|
3
|
-
class Job < QPush::Base::Job
|
4
|
-
def queue
|
5
|
-
Client.redis do |conn|
|
6
|
-
conn.hincrby("#{QPush::Base::KEY}:#{@namespace}:stats", 'queued', 1)
|
7
|
-
conn.lpush("#{QPush::Base::KEY}:#{@namespace}:queue", to_json)
|
8
|
-
end
|
9
|
-
end
|
10
|
-
end
|
11
|
-
end
|
12
|
-
end
|
data/lib/qpush/client/redis.rb
DELETED
@@ -1,16 +0,0 @@
|
|
1
|
-
module QPush
|
2
|
-
module Client
|
3
|
-
class << self
|
4
|
-
def redis
|
5
|
-
redis_pool.with do |conn|
|
6
|
-
yield conn
|
7
|
-
end
|
8
|
-
end
|
9
|
-
|
10
|
-
def redis_pool
|
11
|
-
@redis_pool ||= QPush::Base::RedisPool.create(Client.config.redis_pool,
|
12
|
-
Client.config.redis_url)
|
13
|
-
end
|
14
|
-
end
|
15
|
-
end
|
16
|
-
end
|
data/lib/qpush/server/redis.rb
DELETED
@@ -1,34 +0,0 @@
|
|
1
|
-
module QPush
|
2
|
-
module Server
|
3
|
-
class << self
|
4
|
-
def redis
|
5
|
-
redis_pool.with do |conn|
|
6
|
-
yield conn
|
7
|
-
end
|
8
|
-
end
|
9
|
-
|
10
|
-
def redis_pool
|
11
|
-
@redis_pool ||= QPush::Base::RedisPool.create(Server.config.redis_pool,
|
12
|
-
Server.config.redis_url)
|
13
|
-
end
|
14
|
-
end
|
15
|
-
|
16
|
-
module RedisKeys
|
17
|
-
KEYS = [:delay,
|
18
|
-
:queue,
|
19
|
-
:perform,
|
20
|
-
:stats,
|
21
|
-
:heart,
|
22
|
-
:crons,
|
23
|
-
:history,
|
24
|
-
:morgue]
|
25
|
-
|
26
|
-
def self.build(namespace, priorities)
|
27
|
-
name = "#{QPush::Base::KEY}:#{namespace}"
|
28
|
-
keys = Hash[KEYS.collect { |key| [key, "#{name}:#{key}"] }]
|
29
|
-
keys[:perform_list] = (1..5).collect { |num| "#{keys[:perform]}:#{num}" }
|
30
|
-
keys
|
31
|
-
end
|
32
|
-
end
|
33
|
-
end
|
34
|
-
end
|
data/lib/qpush/web/redis.rb
DELETED
@@ -1,48 +0,0 @@
|
|
1
|
-
module QPush
|
2
|
-
module Web
|
3
|
-
class << self
|
4
|
-
def redis
|
5
|
-
redis_pool.with do |conn|
|
6
|
-
yield conn
|
7
|
-
end
|
8
|
-
end
|
9
|
-
|
10
|
-
def redis_pool
|
11
|
-
@redis_pool ||= QPush::Base::RedisPool.create(Web.config.redis_pool,
|
12
|
-
Web.config.redis_url)
|
13
|
-
end
|
14
|
-
end
|
15
|
-
|
16
|
-
class RedisKeys
|
17
|
-
KEYS = [:delay,
|
18
|
-
:queue,
|
19
|
-
:perform,
|
20
|
-
:stats,
|
21
|
-
:heart,
|
22
|
-
:crons,
|
23
|
-
:history,
|
24
|
-
:morgue]
|
25
|
-
|
26
|
-
attr_reader :delay, :queue, :perform, :stats, :heart,
|
27
|
-
:crons, :history, :morgue
|
28
|
-
|
29
|
-
def initialize(options = {})
|
30
|
-
@namespace = options[:namespace] || 'default'
|
31
|
-
@priorities = options[:priorities] || 5
|
32
|
-
build_keyspaces
|
33
|
-
end
|
34
|
-
|
35
|
-
def perform_list
|
36
|
-
@perform_list ||= (1..@priorities).collect { |num| "#{perform}:#{num}" }
|
37
|
-
end
|
38
|
-
|
39
|
-
private
|
40
|
-
|
41
|
-
def build_keyspaces
|
42
|
-
KEYS.each do |key|
|
43
|
-
instance_variable_set("@#{key}", "#{QPush::Base::KEY}:#{@namespace}:#{key}")
|
44
|
-
end
|
45
|
-
end
|
46
|
-
end
|
47
|
-
end
|
48
|
-
end
|