qpush 0.1.8 → 0.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.byebug_history +0 -242
- data/README.md +4 -2
- data/jobs/fail_job.rb +1 -1
- data/jobs/test_job.rb +2 -2
- data/lib/qpush/base.rb +0 -1
- data/lib/qpush/base/job.rb +1 -0
- data/lib/qpush/base/redis.rb +35 -4
- data/lib/qpush/client.rb +23 -4
- data/lib/qpush/server.rb +2 -3
- data/lib/qpush/server/apis/setup.rb +3 -4
- data/lib/qpush/server/apis/success.rb +1 -3
- data/lib/qpush/server/config.rb +22 -28
- data/lib/qpush/server/jobs.rb +18 -10
- data/lib/qpush/server/launcher.rb +11 -1
- data/lib/qpush/server/logger.rb +0 -2
- data/lib/qpush/server/manager.rb +1 -26
- data/lib/qpush/server/worker.rb +73 -54
- data/lib/qpush/version.rb +1 -1
- data/lib/qpush/web.rb +0 -1
- data/lib/qpush/web/config.rb +1 -4
- metadata +2 -8
- data/dump.rdb +0 -0
- data/lib/qpush/client/config.rb +0 -13
- data/lib/qpush/client/job.rb +0 -12
- data/lib/qpush/client/redis.rb +0 -16
- data/lib/qpush/server/redis.rb +0 -34
- data/lib/qpush/web/redis.rb +0 -48
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: ca96ec0ea37c952b0dc88b82129fae5f60ab604a
|
|
4
|
+
data.tar.gz: f6d0ef10bf31c71e9a8a67d91830fb007500a8dd
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 453061fe039e72dea482d0ba003fab94de6a20ffb18f8076ea0eb68e36fc72c3182018bafb5ee4e2f0c616c9c8cee32be3fa4765103c8b368fe69c269ec0587e
|
|
7
|
+
data.tar.gz: 32bb65924fe5b3189d5f4f7d24d46fc5cda3cd3812458e3f54d5eacc5ccd2fb7cbe147005319468cb1d87565c8fd8eb2d6fd0e06a024444cd44e0ad9eb8417c9
|
data/.byebug_history
CHANGED
|
@@ -1,243 +1 @@
|
|
|
1
1
|
c
|
|
2
|
-
job.valid?
|
|
3
|
-
job.setup
|
|
4
|
-
job
|
|
5
|
-
c
|
|
6
|
-
json
|
|
7
|
-
c
|
|
8
|
-
self.config
|
|
9
|
-
config
|
|
10
|
-
Server.config
|
|
11
|
-
c
|
|
12
|
-
Server.config
|
|
13
|
-
c
|
|
14
|
-
QPush::Server.config
|
|
15
|
-
c
|
|
16
|
-
QPush::Server.config
|
|
17
|
-
c
|
|
18
|
-
job.perform
|
|
19
|
-
job = Job.new(JSON.parse(json))
|
|
20
|
-
json
|
|
21
|
-
c
|
|
22
|
-
delays.count
|
|
23
|
-
delays.any?
|
|
24
|
-
delays
|
|
25
|
-
@conn
|
|
26
|
-
c
|
|
27
|
-
valid?
|
|
28
|
-
c
|
|
29
|
-
configs
|
|
30
|
-
c
|
|
31
|
-
configs
|
|
32
|
-
c
|
|
33
|
-
valid?
|
|
34
|
-
config
|
|
35
|
-
c
|
|
36
|
-
valid?
|
|
37
|
-
@configs
|
|
38
|
-
configs
|
|
39
|
-
c
|
|
40
|
-
configs ? true : false
|
|
41
|
-
configs
|
|
42
|
-
c
|
|
43
|
-
valid?
|
|
44
|
-
quit
|
|
45
|
-
job.klass
|
|
46
|
-
job.errors
|
|
47
|
-
job.valid?
|
|
48
|
-
c
|
|
49
|
-
jobs
|
|
50
|
-
c
|
|
51
|
-
request.path_info
|
|
52
|
-
c
|
|
53
|
-
hash
|
|
54
|
-
c
|
|
55
|
-
@jobs.compact[@start, @count]
|
|
56
|
-
@jobs.compact
|
|
57
|
-
@jobs.compact!
|
|
58
|
-
@count
|
|
59
|
-
@start
|
|
60
|
-
@jobs
|
|
61
|
-
c
|
|
62
|
-
@count
|
|
63
|
-
@start
|
|
64
|
-
@jobs
|
|
65
|
-
c
|
|
66
|
-
Delay.call(@job, :delay)
|
|
67
|
-
c
|
|
68
|
-
Delay.call(@job)
|
|
69
|
-
c
|
|
70
|
-
@stats.each_key { |key| puts key }
|
|
71
|
-
@stats.each_key { |key| @stats[key].to_i }
|
|
72
|
-
@stats
|
|
73
|
-
c
|
|
74
|
-
@stats
|
|
75
|
-
c
|
|
76
|
-
quit
|
|
77
|
-
Perform.call(@job)
|
|
78
|
-
c
|
|
79
|
-
Perform.call(@job)
|
|
80
|
-
c
|
|
81
|
-
@job
|
|
82
|
-
invalid_job && return unless @job.valid?
|
|
83
|
-
c
|
|
84
|
-
job
|
|
85
|
-
c
|
|
86
|
-
@fails = @fails.compact[@start, @count]
|
|
87
|
-
@fails
|
|
88
|
-
@Fails
|
|
89
|
-
@fails.compact!
|
|
90
|
-
@fails.compact![@start..@count]
|
|
91
|
-
@fails.compact![@start, @count]
|
|
92
|
-
@fails.any?
|
|
93
|
-
@count
|
|
94
|
-
@start
|
|
95
|
-
@fails
|
|
96
|
-
c
|
|
97
|
-
@fails
|
|
98
|
-
c
|
|
99
|
-
@fails
|
|
100
|
-
c
|
|
101
|
-
percent
|
|
102
|
-
quit
|
|
103
|
-
y
|
|
104
|
-
q
|
|
105
|
-
(100.00 - ((stats[:failed].to_f / stats[:performed].to_f) * 100.00)).round(2)
|
|
106
|
-
100.00 - ((stats[:failed].to_f / stats[:performed].to_f) * 100.00)
|
|
107
|
-
stats[:failed].to_f / stats[:performed].to_f
|
|
108
|
-
stats[:failed].to_f
|
|
109
|
-
stats[:failed].to_i / stats[:performed].to_i
|
|
110
|
-
(stats[:failed] / stats[:performed])
|
|
111
|
-
stats[:performed]
|
|
112
|
-
stats[:failed]
|
|
113
|
-
percent
|
|
114
|
-
c
|
|
115
|
-
conn.lrange("#{QPush.config.perform_namespace}:last_10", 0, 10)
|
|
116
|
-
c
|
|
117
|
-
json
|
|
118
|
-
c
|
|
119
|
-
conn.get("#{namespace}:#{s}")
|
|
120
|
-
stats
|
|
121
|
-
c
|
|
122
|
-
nil.to_i
|
|
123
|
-
stats
|
|
124
|
-
quit
|
|
125
|
-
Dir[Dir.pwd + "/lib/qpush/jobs/*.rb"]
|
|
126
|
-
Dir[Dir.pwd + "/lib/jobs/*.rb"]
|
|
127
|
-
Dir.pwd
|
|
128
|
-
File.dirname(__FILE__)
|
|
129
|
-
Dir["#{File.dirname(__FILE__)}/qpush/jobs/*.rb"]
|
|
130
|
-
Dir[File.join(QPush.config.jobs_path, "*.rb")]
|
|
131
|
-
$LOAD_PATH.unshift(QPush.config.jobs_path)
|
|
132
|
-
$LOAD_PATH
|
|
133
|
-
File.dirname(__FILE__)
|
|
134
|
-
Dir[File.dirname(__FILE__) + "#{QPush.config.jobs_path}/*.rb"]
|
|
135
|
-
QPush.config.jobs_path
|
|
136
|
-
Dir["#{QPush.config.jobs_path}/*.rb"]
|
|
137
|
-
quit
|
|
138
|
-
puts e
|
|
139
|
-
puts s
|
|
140
|
-
c
|
|
141
|
-
puts e
|
|
142
|
-
puts s
|
|
143
|
-
s
|
|
144
|
-
jobs.compact!
|
|
145
|
-
jobs.compact![s, e]
|
|
146
|
-
jobs
|
|
147
|
-
quit
|
|
148
|
-
jobs
|
|
149
|
-
c
|
|
150
|
-
jobs
|
|
151
|
-
quit
|
|
152
|
-
job_hash
|
|
153
|
-
quit
|
|
154
|
-
delayed_jobs
|
|
155
|
-
delay_jobs
|
|
156
|
-
c
|
|
157
|
-
conn.zrange(QPush.config.delay_namespace, 0, -1, with_scores: true)
|
|
158
|
-
delayed_jobs
|
|
159
|
-
quit
|
|
160
|
-
conn.get("#{@namespace}:#{stat}")
|
|
161
|
-
"#{@namespace}:#{stat}"
|
|
162
|
-
c
|
|
163
|
-
name
|
|
164
|
-
c
|
|
165
|
-
self.name
|
|
166
|
-
quit
|
|
167
|
-
job
|
|
168
|
-
delays.each { |job| PerformJob.call(Job.new(job)) }
|
|
169
|
-
cccccccccccc
|
|
170
|
-
elcccccccccccccccccccc
|
|
171
|
-
delaysdelays.each {|job| PerformJob.call(Job.new(job))}
|
|
172
|
-
quit
|
|
173
|
-
Server.connection_pool
|
|
174
|
-
Qued::QPush.redis
|
|
175
|
-
QPush.redis
|
|
176
|
-
c
|
|
177
|
-
QueueJob.new(job).call
|
|
178
|
-
job.valid? && job.queueable?
|
|
179
|
-
job.valid?
|
|
180
|
-
return false unless job.valid? && job.queueable?
|
|
181
|
-
job
|
|
182
|
-
quit
|
|
183
|
-
@start_at < Time.now.to_i && !@cron.empty?
|
|
184
|
-
@start_at > Time.now.to_i && @cron.empty?
|
|
185
|
-
@start_at < Time.now.to_i && @cron.empty?
|
|
186
|
-
QueueJob.new(self).call
|
|
187
|
-
quit
|
|
188
|
-
job.valid?
|
|
189
|
-
job.save
|
|
190
|
-
test
|
|
191
|
-
quit
|
|
192
|
-
job.save
|
|
193
|
-
job
|
|
194
|
-
job = Job.new(options)
|
|
195
|
-
options
|
|
196
|
-
quit
|
|
197
|
-
cron?
|
|
198
|
-
delay?
|
|
199
|
-
queue?
|
|
200
|
-
quit
|
|
201
|
-
c
|
|
202
|
-
job_type.call
|
|
203
|
-
quit
|
|
204
|
-
job_type.call
|
|
205
|
-
quit
|
|
206
|
-
performer
|
|
207
|
-
performer = retrieve_job_type
|
|
208
|
-
quit
|
|
209
|
-
self
|
|
210
|
-
performer.call
|
|
211
|
-
performer
|
|
212
|
-
performer = retrieve_job_type
|
|
213
|
-
performer.call
|
|
214
|
-
performer
|
|
215
|
-
retrieve_job_type
|
|
216
|
-
performer
|
|
217
|
-
c
|
|
218
|
-
quit
|
|
219
|
-
performer.call
|
|
220
|
-
performer
|
|
221
|
-
performer = retrieve_job_type
|
|
222
|
-
valid?
|
|
223
|
-
c
|
|
224
|
-
@job.queue?
|
|
225
|
-
@job.delay?
|
|
226
|
-
@job.cron?
|
|
227
|
-
@job
|
|
228
|
-
job_success
|
|
229
|
-
measure_run_time { @job.run }
|
|
230
|
-
@job.run
|
|
231
|
-
c
|
|
232
|
-
quit
|
|
233
|
-
log_success
|
|
234
|
-
job_success && log_success
|
|
235
|
-
c
|
|
236
|
-
@job.run
|
|
237
|
-
@job = Job.new(JSON.parse(json))
|
|
238
|
-
c
|
|
239
|
-
@job = Job.new(JSON.parse(json))
|
|
240
|
-
json
|
|
241
|
-
quit
|
|
242
|
-
json
|
|
243
|
-
son
|
data/README.md
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
|
|
1
|
+
#<center><img src="https://s21.postimg.org/85ce7g93b/qpush.png" alt="Drawing" width="250"/></center>
|
|
2
2
|
[](https://codeclimate.com/github/nsweeting/qpush)
|
|
3
3
|
|
|
4
4
|
Fast and simple job queue microservice for Ruby. **Please consider it under development at the moment.**
|
|
@@ -77,12 +77,14 @@ At a minimum, we must provide the job with a 'klass'. There are many more option
|
|
|
77
77
|
|
|
78
78
|
#### Building Jobs
|
|
79
79
|
|
|
80
|
-
Jobs are simply
|
|
80
|
+
Jobs are simply ruby objects that include the QPush::Job module and contain a 'call' method. If you provide a hash for the 'args' of the job, the job will be initialized with them. Below is an example of a simple mailing job utilizing the 'mail' gem.
|
|
81
81
|
|
|
82
82
|
```ruby
|
|
83
83
|
require 'mail'
|
|
84
84
|
|
|
85
85
|
class MailJob
|
|
86
|
+
include QPush::Job
|
|
87
|
+
|
|
86
88
|
def initialize(options = {})
|
|
87
89
|
@mail = Mail.new(options)
|
|
88
90
|
end
|
data/jobs/fail_job.rb
CHANGED
data/jobs/test_job.rb
CHANGED
data/lib/qpush/base.rb
CHANGED
data/lib/qpush/base/job.rb
CHANGED
data/lib/qpush/base/redis.rb
CHANGED
|
@@ -1,11 +1,42 @@
|
|
|
1
1
|
module QPush
|
|
2
2
|
module Base
|
|
3
3
|
KEY = 'qpush:v1'.freeze
|
|
4
|
+
SUB_KEYS = [:delay,
|
|
5
|
+
:queue,
|
|
6
|
+
:perform,
|
|
7
|
+
:stats,
|
|
8
|
+
:heart,
|
|
9
|
+
:crons,
|
|
10
|
+
:history,
|
|
11
|
+
:morgue].freeze
|
|
4
12
|
|
|
5
|
-
|
|
6
|
-
def self.
|
|
7
|
-
|
|
8
|
-
|
|
13
|
+
module RedisHelper
|
|
14
|
+
def self.included(base)
|
|
15
|
+
base.extend(ClassMethods)
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
module ClassMethods
|
|
19
|
+
def redis
|
|
20
|
+
redis_pool.with do |conn|
|
|
21
|
+
yield conn
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def redis_pool
|
|
26
|
+
@redis_pool ||= build_pool(config.redis_pool, config.redis_url)
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def build_pool(pool, url)
|
|
30
|
+
ConnectionPool.new(size: pool) do
|
|
31
|
+
Redis.new(url: url)
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def build_keys(namespace, priorities)
|
|
36
|
+
name = "#{QPush::Base::KEY}:#{namespace}"
|
|
37
|
+
keys = Hash[QPush::Base::SUB_KEYS.collect { |key| [key, "#{name}:#{key}"] }]
|
|
38
|
+
keys[:perform_list] = (1..priorities).collect { |num| "#{keys[:perform]}:#{num}" }
|
|
39
|
+
keys
|
|
9
40
|
end
|
|
10
41
|
end
|
|
11
42
|
end
|
data/lib/qpush/client.rb
CHANGED
|
@@ -1,7 +1,26 @@
|
|
|
1
1
|
# QPush Base
|
|
2
2
|
require 'qpush/base'
|
|
3
3
|
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
4
|
+
module QPush
|
|
5
|
+
module Client
|
|
6
|
+
include QPush::Base::ConfigHelper
|
|
7
|
+
include QPush::Base::RedisHelper
|
|
8
|
+
|
|
9
|
+
class << self
|
|
10
|
+
def config
|
|
11
|
+
@config ||= Config.new
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
class Config < QPush::Base::Config; end
|
|
16
|
+
|
|
17
|
+
class Job < QPush::Base::Job
|
|
18
|
+
def queue
|
|
19
|
+
Client.redis do |conn|
|
|
20
|
+
conn.hincrby("#{QPush::Base::KEY}:#{@namespace}:stats", 'queued', 1)
|
|
21
|
+
conn.lpush("#{QPush::Base::KEY}:#{@namespace}:queue", to_json)
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
data/lib/qpush/server.rb
CHANGED
|
@@ -2,13 +2,14 @@
|
|
|
2
2
|
require 'sequel'
|
|
3
3
|
require 'object_validator'
|
|
4
4
|
require 'parse-cron'
|
|
5
|
-
require '
|
|
5
|
+
require 'logger'
|
|
6
6
|
|
|
7
7
|
# QPush Base
|
|
8
8
|
require 'qpush/base'
|
|
9
9
|
|
|
10
10
|
# Qpush Server Base
|
|
11
11
|
require 'qpush/server/apis'
|
|
12
|
+
require 'qpush/server/worker'
|
|
12
13
|
require 'qpush/server/config'
|
|
13
14
|
require 'qpush/server/database'
|
|
14
15
|
require 'qpush/server/delay'
|
|
@@ -21,8 +22,6 @@ require 'qpush/server/logger'
|
|
|
21
22
|
require 'qpush/server/manager'
|
|
22
23
|
require 'qpush/server/perform'
|
|
23
24
|
require 'qpush/server/queue'
|
|
24
|
-
require 'qpush/server/redis'
|
|
25
|
-
require 'qpush/server/worker'
|
|
26
25
|
|
|
27
26
|
# QPush Server Apis
|
|
28
27
|
require 'qpush/server/apis/delay'
|
|
@@ -3,15 +3,14 @@ module QPush
|
|
|
3
3
|
module Apis
|
|
4
4
|
class Setup < Base
|
|
5
5
|
def call
|
|
6
|
-
|
|
7
|
-
setup_job
|
|
6
|
+
@job.valid? ? setup_job : invalid_job
|
|
8
7
|
end
|
|
9
8
|
|
|
10
9
|
private
|
|
11
10
|
|
|
12
11
|
def setup_job
|
|
13
|
-
|
|
14
|
-
|
|
12
|
+
@job.perform if @job.perform_job?
|
|
13
|
+
@job.delay if @job.delay_job?
|
|
15
14
|
end
|
|
16
15
|
|
|
17
16
|
def invalid_job
|
data/lib/qpush/server/config.rb
CHANGED
|
@@ -1,43 +1,19 @@
|
|
|
1
1
|
module QPush
|
|
2
2
|
module Server
|
|
3
3
|
include QPush::Base::ConfigHelper
|
|
4
|
+
include QPush::Base::RedisHelper
|
|
4
5
|
|
|
5
6
|
class << self
|
|
6
|
-
attr_accessor :
|
|
7
|
+
attr_accessor :keys
|
|
7
8
|
|
|
8
9
|
def config
|
|
9
10
|
@config ||= Config.new
|
|
10
11
|
end
|
|
11
|
-
|
|
12
|
-
def build_worker
|
|
13
|
-
worker = WorkerConfig.new
|
|
14
|
-
yield worker
|
|
15
|
-
worker
|
|
16
|
-
end
|
|
17
|
-
end
|
|
18
|
-
|
|
19
|
-
class WorkerConfig
|
|
20
|
-
DEFAULTS = {
|
|
21
|
-
namespace: 'default',
|
|
22
|
-
priorities: 5,
|
|
23
|
-
queue_threads: 2,
|
|
24
|
-
perform_threads: 2,
|
|
25
|
-
delay_threads: 1 }.freeze
|
|
26
|
-
|
|
27
|
-
attr_accessor :perform_threads, :queue_threads, :delay_threads,
|
|
28
|
-
:namespace, :priorities
|
|
29
|
-
|
|
30
|
-
def initialize(options = {})
|
|
31
|
-
options = DEFAULTS.merge(options)
|
|
32
|
-
options.each { |key, value| send("#{key}=", value) }
|
|
33
|
-
end
|
|
34
|
-
|
|
35
|
-
def for_keys
|
|
36
|
-
{ namespace: @namespace, priorities: @priorities }
|
|
37
|
-
end
|
|
38
12
|
end
|
|
39
13
|
|
|
40
14
|
class Config < QPush::Base::Config
|
|
15
|
+
include ObjectValidator::Validate
|
|
16
|
+
|
|
41
17
|
SERVER_DEFAULTS = {
|
|
42
18
|
database_url: ENV['DATABASE_URL'],
|
|
43
19
|
database_pool: 10,
|
|
@@ -50,6 +26,24 @@ module QPush
|
|
|
50
26
|
super
|
|
51
27
|
SERVER_DEFAULTS.each { |key, value| send("#{key}=", value) }
|
|
52
28
|
end
|
|
29
|
+
|
|
30
|
+
def validate!
|
|
31
|
+
return if valid?
|
|
32
|
+
fail ServerError, errors.full_messages.join(' ')
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
class ConfigValidator
|
|
37
|
+
include ObjectValidator::Validator
|
|
38
|
+
|
|
39
|
+
validates :redis, with: { proc: proc { Server.redis { |c| c.ping && c.quit } },
|
|
40
|
+
msg: 'could not be connected with' }
|
|
41
|
+
validates :workers, with: { proc: proc { |c| c.workers.is_a?(Array) && c.workers.count > 0 },
|
|
42
|
+
msg: 'is not a valid Array of WorkerConfigs' }
|
|
43
|
+
validates :configs, with: { proc: proc { |c| c.workers.all? { |w| w.is_a?(WorkerConfig) } },
|
|
44
|
+
msg: 'are not valid WorkerConfig objects' }
|
|
45
|
+
validates :jobs_path, with: { proc: proc { |c| Dir.exist?(Dir.pwd + c.jobs_path) },
|
|
46
|
+
msg: 'is not a valid directory' }
|
|
53
47
|
end
|
|
54
48
|
end
|
|
55
49
|
end
|
data/lib/qpush/server/jobs.rb
CHANGED
|
@@ -1,14 +1,22 @@
|
|
|
1
1
|
module QPush
|
|
2
|
+
module Job
|
|
3
|
+
class << self
|
|
4
|
+
def included(base)
|
|
5
|
+
_register_job(base)
|
|
6
|
+
Server.log.info("* Job loaded: #{base.name}")
|
|
7
|
+
end
|
|
8
|
+
|
|
9
|
+
def _register_job(base)
|
|
10
|
+
Server.jobs << base.name
|
|
11
|
+
Server.redis { |c| c.sadd("#{QPush::Base::KEY}:jobs", base.name) }
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
end
|
|
15
|
+
|
|
2
16
|
module Server
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
_register_job(base)
|
|
7
|
-
end
|
|
8
|
-
|
|
9
|
-
def _register_job(base)
|
|
10
|
-
Server.redis { |c| c.sadd("#{QPush::Base::KEY}:jobs", base.name) }
|
|
11
|
-
end
|
|
17
|
+
class << self
|
|
18
|
+
def jobs
|
|
19
|
+
@jobs ||= []
|
|
12
20
|
end
|
|
13
21
|
end
|
|
14
22
|
|
|
@@ -69,7 +77,7 @@ module QPush
|
|
|
69
77
|
include ObjectValidator::Validator
|
|
70
78
|
|
|
71
79
|
validates :klass,
|
|
72
|
-
with: { proc: proc { |j|
|
|
80
|
+
with: { proc: proc { |j| Server.jobs.include?(j.klass) },
|
|
73
81
|
msg: 'has not been defined' }
|
|
74
82
|
validates :cron,
|
|
75
83
|
with: { proc: proc { |j| j.cron.empty? ? true : CronParser.new(j.cron) },
|
|
@@ -7,11 +7,12 @@ module QPush
|
|
|
7
7
|
@argv = argv
|
|
8
8
|
end
|
|
9
9
|
|
|
10
|
-
#
|
|
10
|
+
# Provides the main entrypoint for starting a QPush server.
|
|
11
11
|
#
|
|
12
12
|
def start
|
|
13
13
|
start_message
|
|
14
14
|
setup_options
|
|
15
|
+
validate!
|
|
15
16
|
setup_jobs
|
|
16
17
|
boot_manager
|
|
17
18
|
end
|
|
@@ -39,12 +40,21 @@ module QPush
|
|
|
39
40
|
parser.parse!(@argv)
|
|
40
41
|
end
|
|
41
42
|
|
|
43
|
+
# Validates our server and worker configuration.
|
|
44
|
+
#
|
|
45
|
+
def validate!
|
|
46
|
+
Server.config.validate!
|
|
47
|
+
Server.config.workers.each { |w| w.validate! }
|
|
48
|
+
end
|
|
49
|
+
|
|
42
50
|
# Requires all base jobs as well as user jobs.
|
|
43
51
|
#
|
|
44
52
|
def setup_jobs
|
|
45
53
|
JobLoader.call
|
|
46
54
|
end
|
|
47
55
|
|
|
56
|
+
# Boots our manager
|
|
57
|
+
#
|
|
48
58
|
def boot_manager
|
|
49
59
|
manager = Manager.new(Server.config.workers)
|
|
50
60
|
manager.start
|
data/lib/qpush/server/logger.rb
CHANGED
data/lib/qpush/server/manager.rb
CHANGED
|
@@ -4,8 +4,6 @@ module QPush
|
|
|
4
4
|
# of them to start and shutdown.
|
|
5
5
|
#
|
|
6
6
|
class Manager
|
|
7
|
-
include ObjectValidator::Validate
|
|
8
|
-
|
|
9
7
|
attr_accessor :configs
|
|
10
8
|
attr_reader :forks
|
|
11
9
|
|
|
@@ -21,7 +19,6 @@ module QPush
|
|
|
21
19
|
# sleep so that our Workers can do their thing.
|
|
22
20
|
#
|
|
23
21
|
def start
|
|
24
|
-
validate!
|
|
25
22
|
start_messages
|
|
26
23
|
flush_spaces
|
|
27
24
|
create_workers
|
|
@@ -55,34 +52,12 @@ module QPush
|
|
|
55
52
|
Server.log.info("* Worker count: #{@configs.count}")
|
|
56
53
|
end
|
|
57
54
|
|
|
58
|
-
# Validates our data before starting our Workers. Also instantiates our
|
|
59
|
-
# connection pool by pinging Redis.
|
|
60
|
-
#
|
|
61
|
-
def validate!
|
|
62
|
-
return if valid?
|
|
63
|
-
fail ServerError, errors.full_messages.join(' ')
|
|
64
|
-
end
|
|
65
|
-
|
|
66
55
|
# Removes the list of namespaces used by our server from Redis. This
|
|
67
56
|
# prepares it for the new list that will be created by our workers.
|
|
68
57
|
#
|
|
69
58
|
def flush_spaces
|
|
70
|
-
Server.redis { |c| c.del(QPush::Base::KEY
|
|
59
|
+
Server.redis { |c| c.del("#{QPush::Base::KEY}:namespaces") }
|
|
71
60
|
end
|
|
72
61
|
end
|
|
73
|
-
|
|
74
|
-
# The ManagerValidator ensures the data for our manager is valid before
|
|
75
|
-
# attempting to start it.
|
|
76
|
-
#
|
|
77
|
-
class ManagerValidator
|
|
78
|
-
include ObjectValidator::Validator
|
|
79
|
-
|
|
80
|
-
validates :redis, with: { proc: proc { Server.redis { |c| c.ping && c.quit } },
|
|
81
|
-
msg: 'could not be connected with' }
|
|
82
|
-
validates :configs, with: { proc: proc { |m| m.configs.count > 0 },
|
|
83
|
-
msg: 'were not defined' }
|
|
84
|
-
validates :configs, with: { proc: proc { |m| m.configs.each { |c| c.is_a?(WorkerConfig) } },
|
|
85
|
-
msg: 'are not valid WorkerConfig objects' }
|
|
86
|
-
end
|
|
87
62
|
end
|
|
88
63
|
end
|
data/lib/qpush/server/worker.rb
CHANGED
|
@@ -1,20 +1,26 @@
|
|
|
1
|
-
|
|
2
1
|
module QPush
|
|
3
2
|
module Server
|
|
4
|
-
|
|
5
|
-
|
|
3
|
+
class << self
|
|
4
|
+
attr_accessor :worker
|
|
5
|
+
|
|
6
|
+
# A convenience method used to create new WorkerConfig objects for use
|
|
7
|
+
# in our server configuration.
|
|
8
|
+
#
|
|
9
|
+
def build_worker
|
|
10
|
+
worker = WorkerConfig.new
|
|
11
|
+
yield worker
|
|
12
|
+
worker
|
|
13
|
+
end
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
# The Worker manages our actions - Queue, Delay, Perform and Heartbeat.
|
|
17
|
+
# Each of these actions is alloted a number of threads. Each action
|
|
6
18
|
# object maintains control of these threads through the aptly named start
|
|
7
19
|
# and shutdown methods.
|
|
8
20
|
#
|
|
9
21
|
class Worker
|
|
10
|
-
extend Forwardable
|
|
11
|
-
include ObjectValidator::Validate
|
|
12
|
-
|
|
13
22
|
attr_reader :config, :pid, :id
|
|
14
23
|
|
|
15
|
-
def_delegators :@config, :perform_threads, :delay_threads, :queue_threads,
|
|
16
|
-
:priorities, :base_threads, :namespace
|
|
17
|
-
|
|
18
24
|
def initialize(id, config)
|
|
19
25
|
@id = id
|
|
20
26
|
@pid = Process.pid
|
|
@@ -27,11 +33,10 @@ module QPush
|
|
|
27
33
|
# Starts our new worker.
|
|
28
34
|
#
|
|
29
35
|
def start
|
|
30
|
-
validate!
|
|
31
36
|
assign_globals
|
|
32
37
|
register_space
|
|
33
38
|
start_message
|
|
34
|
-
|
|
39
|
+
build_actions
|
|
35
40
|
start_threads
|
|
36
41
|
end
|
|
37
42
|
|
|
@@ -45,42 +50,56 @@ module QPush
|
|
|
45
50
|
|
|
46
51
|
private
|
|
47
52
|
|
|
48
|
-
#
|
|
49
|
-
# our Queue and Retry objects. We then start them and join them to the
|
|
50
|
-
# main process.
|
|
53
|
+
# Assign the globals that are required for our worker to function.
|
|
51
54
|
#
|
|
52
|
-
def
|
|
53
|
-
@
|
|
54
|
-
|
|
55
|
+
def assign_globals
|
|
56
|
+
Server.keys = Server.build_keys(@config.namespace, @config.priorities)
|
|
57
|
+
Server.worker = self
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# Registers our workers namespace on Redis
|
|
61
|
+
#
|
|
62
|
+
def register_space
|
|
63
|
+
Server.redis do |c|
|
|
64
|
+
c.sadd("#{QPush::Base::KEY}:namespaces", @config.namespace)
|
|
55
65
|
end
|
|
56
|
-
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# Information about the start process
|
|
69
|
+
#
|
|
70
|
+
def start_message
|
|
71
|
+
Server.log.info("* Worker #{@id} started | pid: #{@pid} | namespace: #{@config.namespace}")
|
|
57
72
|
end
|
|
58
73
|
|
|
59
74
|
# Instantiates our Queue, Perform, Delay and Heartbeat objects based on
|
|
60
|
-
# the number of threads specified for each
|
|
75
|
+
# the number of threads specified for each action type. We store these
|
|
61
76
|
# objects as an array in @actions.
|
|
62
77
|
#
|
|
63
|
-
def
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
@actions <<
|
|
78
|
+
def build_actions
|
|
79
|
+
base_actions.each do |action|
|
|
80
|
+
action[:count].times do
|
|
81
|
+
@actions << action[:klass].new
|
|
67
82
|
end
|
|
68
83
|
end
|
|
69
84
|
end
|
|
70
85
|
|
|
71
|
-
def
|
|
86
|
+
def base_actions
|
|
72
87
|
[
|
|
73
|
-
{ klass: Perform, count: perform_threads },
|
|
74
|
-
{ klass: Queue, count: queue_threads },
|
|
75
|
-
{ klass: Delay, count: delay_threads },
|
|
88
|
+
{ klass: Perform, count: @config.perform_threads },
|
|
89
|
+
{ klass: Queue, count: @config.queue_threads },
|
|
90
|
+
{ klass: Delay, count: @config.delay_threads },
|
|
76
91
|
{ klass: Heartbeat, count: 1 }
|
|
77
92
|
]
|
|
78
93
|
end
|
|
79
94
|
|
|
80
|
-
#
|
|
95
|
+
# Creates threads for each of the action objects, We then start them and
|
|
96
|
+
# join them to the main process.
|
|
81
97
|
#
|
|
82
|
-
def
|
|
83
|
-
|
|
98
|
+
def start_threads
|
|
99
|
+
@actions.each do |action|
|
|
100
|
+
@threads << Thread.new { action.start }
|
|
101
|
+
end
|
|
102
|
+
@threads.map(&:join)
|
|
84
103
|
end
|
|
85
104
|
|
|
86
105
|
# Information about the shutdown process
|
|
@@ -88,40 +107,40 @@ module QPush
|
|
|
88
107
|
def shutdown_message
|
|
89
108
|
Server.log.info("* Worker #{@id} shutdown | pid: #{@pid}")
|
|
90
109
|
end
|
|
110
|
+
end
|
|
91
111
|
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
def validate!
|
|
95
|
-
return if valid?
|
|
96
|
-
fail ServerError, errors.full_messages.join(' ')
|
|
97
|
-
end
|
|
112
|
+
class WorkerConfig
|
|
113
|
+
include ObjectValidator::Validate
|
|
98
114
|
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
115
|
+
DEFAULTS = {
|
|
116
|
+
namespace: 'default',
|
|
117
|
+
priorities: 5,
|
|
118
|
+
queue_threads: 2,
|
|
119
|
+
perform_threads: 2,
|
|
120
|
+
delay_threads: 1 }.freeze
|
|
121
|
+
|
|
122
|
+
attr_accessor :perform_threads, :queue_threads, :delay_threads,
|
|
123
|
+
:namespace, :priorities
|
|
124
|
+
|
|
125
|
+
def initialize(options = {})
|
|
126
|
+
options = DEFAULTS.merge(options)
|
|
127
|
+
options.each { |key, value| send("#{key}=", value) }
|
|
102
128
|
end
|
|
103
129
|
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
Server.redis do |c|
|
|
108
|
-
c.sadd(QPush::Base::KEY + ':namespaces', namespace)
|
|
109
|
-
end
|
|
130
|
+
def validate!
|
|
131
|
+
return if valid?
|
|
132
|
+
fail ServerError, errors.full_messages.join(' ')
|
|
110
133
|
end
|
|
111
134
|
end
|
|
112
135
|
|
|
113
|
-
|
|
114
|
-
# valid before attempting to use it.
|
|
115
|
-
#
|
|
116
|
-
class WorkerValidator
|
|
136
|
+
class WorkerConfigValidator
|
|
117
137
|
include ObjectValidator::Validator
|
|
118
138
|
|
|
119
|
-
validates :config, type: QPush::Server::WorkerConfig
|
|
120
|
-
validates :perform_threads, type: Integer, greater_than: 0
|
|
121
|
-
validates :queue_threads, type: Integer, greater_than: 0
|
|
122
|
-
validates :delay_threads, type: Integer, greater_than: 0
|
|
123
139
|
validates :namespace, type: String
|
|
124
|
-
validates :priorities,
|
|
140
|
+
validates :priorities, greater_than: 4
|
|
141
|
+
validates :queue_threads, greater_than: 0
|
|
142
|
+
validates :perform_threads, greater_than: 0
|
|
143
|
+
validates :delay_threads, greater_than: 0
|
|
125
144
|
end
|
|
126
145
|
end
|
|
127
146
|
end
|
data/lib/qpush/version.rb
CHANGED
data/lib/qpush/web.rb
CHANGED
data/lib/qpush/web/config.rb
CHANGED
|
@@ -1,15 +1,12 @@
|
|
|
1
1
|
module QPush
|
|
2
2
|
module Web
|
|
3
3
|
include QPush::Base::ConfigHelper
|
|
4
|
+
include QPush::Base::RedisHelper
|
|
4
5
|
|
|
5
6
|
class << self
|
|
6
7
|
def config
|
|
7
8
|
@config ||= Config.new
|
|
8
9
|
end
|
|
9
|
-
|
|
10
|
-
def keys
|
|
11
|
-
@keys ||= QPush::Web::RedisKeys.new
|
|
12
|
-
end
|
|
13
10
|
end
|
|
14
11
|
|
|
15
12
|
class Config < QPush::Base::Config; end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: qpush
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.10
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Nicholas Sweeting
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2016-08-
|
|
11
|
+
date: 2016-08-28 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: redis
|
|
@@ -200,7 +200,6 @@ files:
|
|
|
200
200
|
- bin/qpush-server
|
|
201
201
|
- bin/qpush-web
|
|
202
202
|
- bin/setup
|
|
203
|
-
- dump.rdb
|
|
204
203
|
- jobs/fail_job.rb
|
|
205
204
|
- jobs/test_job.rb
|
|
206
205
|
- lib/qpush.rb
|
|
@@ -209,9 +208,6 @@ files:
|
|
|
209
208
|
- lib/qpush/base/job.rb
|
|
210
209
|
- lib/qpush/base/redis.rb
|
|
211
210
|
- lib/qpush/client.rb
|
|
212
|
-
- lib/qpush/client/config.rb
|
|
213
|
-
- lib/qpush/client/job.rb
|
|
214
|
-
- lib/qpush/client/redis.rb
|
|
215
211
|
- lib/qpush/jobs/queue_delayed.rb
|
|
216
212
|
- lib/qpush/server.rb
|
|
217
213
|
- lib/qpush/server/apis.rb
|
|
@@ -236,7 +232,6 @@ files:
|
|
|
236
232
|
- lib/qpush/server/manager.rb
|
|
237
233
|
- lib/qpush/server/perform.rb
|
|
238
234
|
- lib/qpush/server/queue.rb
|
|
239
|
-
- lib/qpush/server/redis.rb
|
|
240
235
|
- lib/qpush/server/worker.rb
|
|
241
236
|
- lib/qpush/version.rb
|
|
242
237
|
- lib/qpush/web.rb
|
|
@@ -267,7 +262,6 @@ files:
|
|
|
267
262
|
- lib/qpush/web/public/js/app.js
|
|
268
263
|
- lib/qpush/web/public/js/shims.js
|
|
269
264
|
- lib/qpush/web/public/tsconfig.json
|
|
270
|
-
- lib/qpush/web/redis.rb
|
|
271
265
|
- lib/qpush/web/server.rb
|
|
272
266
|
- lib/qpush/web/server.ru
|
|
273
267
|
- qpush.gemspec
|
data/dump.rdb
DELETED
|
Binary file
|
data/lib/qpush/client/config.rb
DELETED
data/lib/qpush/client/job.rb
DELETED
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
module QPush
|
|
2
|
-
module Client
|
|
3
|
-
class Job < QPush::Base::Job
|
|
4
|
-
def queue
|
|
5
|
-
Client.redis do |conn|
|
|
6
|
-
conn.hincrby("#{QPush::Base::KEY}:#{@namespace}:stats", 'queued', 1)
|
|
7
|
-
conn.lpush("#{QPush::Base::KEY}:#{@namespace}:queue", to_json)
|
|
8
|
-
end
|
|
9
|
-
end
|
|
10
|
-
end
|
|
11
|
-
end
|
|
12
|
-
end
|
data/lib/qpush/client/redis.rb
DELETED
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
module QPush
|
|
2
|
-
module Client
|
|
3
|
-
class << self
|
|
4
|
-
def redis
|
|
5
|
-
redis_pool.with do |conn|
|
|
6
|
-
yield conn
|
|
7
|
-
end
|
|
8
|
-
end
|
|
9
|
-
|
|
10
|
-
def redis_pool
|
|
11
|
-
@redis_pool ||= QPush::Base::RedisPool.create(Client.config.redis_pool,
|
|
12
|
-
Client.config.redis_url)
|
|
13
|
-
end
|
|
14
|
-
end
|
|
15
|
-
end
|
|
16
|
-
end
|
data/lib/qpush/server/redis.rb
DELETED
|
@@ -1,34 +0,0 @@
|
|
|
1
|
-
module QPush
|
|
2
|
-
module Server
|
|
3
|
-
class << self
|
|
4
|
-
def redis
|
|
5
|
-
redis_pool.with do |conn|
|
|
6
|
-
yield conn
|
|
7
|
-
end
|
|
8
|
-
end
|
|
9
|
-
|
|
10
|
-
def redis_pool
|
|
11
|
-
@redis_pool ||= QPush::Base::RedisPool.create(Server.config.redis_pool,
|
|
12
|
-
Server.config.redis_url)
|
|
13
|
-
end
|
|
14
|
-
end
|
|
15
|
-
|
|
16
|
-
module RedisKeys
|
|
17
|
-
KEYS = [:delay,
|
|
18
|
-
:queue,
|
|
19
|
-
:perform,
|
|
20
|
-
:stats,
|
|
21
|
-
:heart,
|
|
22
|
-
:crons,
|
|
23
|
-
:history,
|
|
24
|
-
:morgue]
|
|
25
|
-
|
|
26
|
-
def self.build(namespace, priorities)
|
|
27
|
-
name = "#{QPush::Base::KEY}:#{namespace}"
|
|
28
|
-
keys = Hash[KEYS.collect { |key| [key, "#{name}:#{key}"] }]
|
|
29
|
-
keys[:perform_list] = (1..5).collect { |num| "#{keys[:perform]}:#{num}" }
|
|
30
|
-
keys
|
|
31
|
-
end
|
|
32
|
-
end
|
|
33
|
-
end
|
|
34
|
-
end
|
data/lib/qpush/web/redis.rb
DELETED
|
@@ -1,48 +0,0 @@
|
|
|
1
|
-
module QPush
|
|
2
|
-
module Web
|
|
3
|
-
class << self
|
|
4
|
-
def redis
|
|
5
|
-
redis_pool.with do |conn|
|
|
6
|
-
yield conn
|
|
7
|
-
end
|
|
8
|
-
end
|
|
9
|
-
|
|
10
|
-
def redis_pool
|
|
11
|
-
@redis_pool ||= QPush::Base::RedisPool.create(Web.config.redis_pool,
|
|
12
|
-
Web.config.redis_url)
|
|
13
|
-
end
|
|
14
|
-
end
|
|
15
|
-
|
|
16
|
-
class RedisKeys
|
|
17
|
-
KEYS = [:delay,
|
|
18
|
-
:queue,
|
|
19
|
-
:perform,
|
|
20
|
-
:stats,
|
|
21
|
-
:heart,
|
|
22
|
-
:crons,
|
|
23
|
-
:history,
|
|
24
|
-
:morgue]
|
|
25
|
-
|
|
26
|
-
attr_reader :delay, :queue, :perform, :stats, :heart,
|
|
27
|
-
:crons, :history, :morgue
|
|
28
|
-
|
|
29
|
-
def initialize(options = {})
|
|
30
|
-
@namespace = options[:namespace] || 'default'
|
|
31
|
-
@priorities = options[:priorities] || 5
|
|
32
|
-
build_keyspaces
|
|
33
|
-
end
|
|
34
|
-
|
|
35
|
-
def perform_list
|
|
36
|
-
@perform_list ||= (1..@priorities).collect { |num| "#{perform}:#{num}" }
|
|
37
|
-
end
|
|
38
|
-
|
|
39
|
-
private
|
|
40
|
-
|
|
41
|
-
def build_keyspaces
|
|
42
|
-
KEYS.each do |key|
|
|
43
|
-
instance_variable_set("@#{key}", "#{QPush::Base::KEY}:#{@namespace}:#{key}")
|
|
44
|
-
end
|
|
45
|
-
end
|
|
46
|
-
end
|
|
47
|
-
end
|
|
48
|
-
end
|