qpush 0.1.1 → 0.1.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.byebug_history +13 -0
- data/Gemfile +1 -1
- data/README.md +3 -3
- data/{lib/qpush/jobs → jobs}/test_job.rb +1 -1
- data/lib/qpush/base.rb +0 -2
- data/lib/qpush/config.rb +23 -10
- data/lib/qpush/job.rb +26 -18
- data/lib/qpush/server/execute.rb +12 -4
- data/lib/qpush/server/jobs.rb +22 -34
- data/lib/qpush/server/launcher.rb +7 -0
- data/lib/qpush/version.rb +1 -1
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 01a15d5df46ac29fd256af5b940ea82249588729
|
4
|
+
data.tar.gz: 62a42bc427451e3d47f0d2a3ac38eaafcdee4020
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 283c6796ce8f178a7a5cb3722f73716c31bc50bd3522dd68ff74a70b597a7ba728094fdd688e70f74fdb4ae3f9aff8e943209f878ab60f81198fbd39a344af83
|
7
|
+
data.tar.gz: df33a1a4d1fd0c9efd339c17866f4db22ad0c7d05b62d6ee92f656f9adb94d6cf646fa937cca0d3f942a28fb9768cdbd1a8b7f1303e78c191639999e384fcdd4
|
data/.byebug_history
CHANGED
@@ -1,4 +1,17 @@
|
|
1
1
|
quit
|
2
|
+
Dir[Dir.pwd + "/lib/qpush/jobs/*.rb"]
|
3
|
+
Dir[Dir.pwd + "/lib/jobs/*.rb"]
|
4
|
+
Dir.pwd
|
5
|
+
File.dirname(__FILE__)
|
6
|
+
Dir["#{File.dirname(__FILE__)}/qpush/jobs/*.rb"]
|
7
|
+
Dir[File.join(QPush.config.jobs_path, "*.rb")]
|
8
|
+
$LOAD_PATH.unshift(QPush.config.jobs_path)
|
9
|
+
$LOAD_PATH
|
10
|
+
File.dirname(__FILE__)
|
11
|
+
Dir[File.dirname(__FILE__) + "#{QPush.config.jobs_path}/*.rb"]
|
12
|
+
QPush.config.jobs_path
|
13
|
+
Dir["#{QPush.config.jobs_path}/*.rb"]
|
14
|
+
quit
|
2
15
|
puts e
|
3
16
|
puts s
|
4
17
|
c
|
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -33,7 +33,7 @@ In order to process queued jobs, we run the QPush server. This is a separate ser
|
|
33
33
|
|
34
34
|
$ bundle exec qpush-server -c path/to/config.rb
|
35
35
|
|
36
|
-
By providing a path to a configuration file, we can setup QPush with plain old ruby. At a minimum, we should provide details on our Redis server and connections. There are more configuration options available - all of which can be viewed here
|
36
|
+
By providing a path to a configuration file, we can setup QPush with plain old ruby. At a minimum, we should provide details on our Redis server and connections. There are more configuration options available - [all of which can be viewed here](https://github.com/nsweeting/qpush/wiki/Server-Configuration).
|
37
37
|
|
38
38
|
```ruby
|
39
39
|
# QPush server configuration
|
@@ -73,7 +73,7 @@ The job above would be equivalent to running the following command on the server
|
|
73
73
|
Example::Job.new(example: 'Job').call
|
74
74
|
```
|
75
75
|
|
76
|
-
At a minimum, we must provide the job with a 'klass'. There are many more options that we can provide to the job though - all of which can be viewed here
|
76
|
+
At a minimum, we must provide the job with a 'klass'. There are many more options that we can provide to the job though - [all of which can be viewed here](https://github.com/nsweeting/qpush/wiki/Options-for-Jobs).
|
77
77
|
|
78
78
|
#### Building Jobs
|
79
79
|
|
@@ -146,7 +146,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
|
|
146
146
|
|
147
147
|
## Contributing
|
148
148
|
|
149
|
-
Bug reports and pull requests are welcome on GitHub at https://github.com/
|
149
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/nsweeting/qpush. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
|
150
150
|
|
151
151
|
|
152
152
|
## License
|
data/lib/qpush/base.rb
CHANGED
data/lib/qpush/config.rb
CHANGED
@@ -22,20 +22,17 @@ module QPush
|
|
22
22
|
redis_pool: 10,
|
23
23
|
database_pool: 10,
|
24
24
|
workers: 2,
|
25
|
-
|
25
|
+
namespace: 'default',
|
26
26
|
queue_threads: 2,
|
27
|
-
queue_namespace: 'qpush:v1:queue',
|
28
27
|
perform_threads: 2,
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
priorities: 5
|
28
|
+
delay_threads: 1,
|
29
|
+
priorities: 5,
|
30
|
+
jobs_path: '/jobs'
|
33
31
|
}.freeze
|
34
32
|
|
35
|
-
attr_accessor :workers, :queue_threads, :
|
36
|
-
:
|
37
|
-
:
|
38
|
-
:database_url, :database_pool, :database_adapter
|
33
|
+
attr_accessor :workers, :queue_threads, :namespace, :delay_threads,
|
34
|
+
:perform_threads, :redis_url, :redis_pool, :priorities,
|
35
|
+
:database_url, :database_pool, :jobs_path
|
39
36
|
|
40
37
|
def initialize
|
41
38
|
DEFAULTS.each { |key, value| send("#{key}=", value) }
|
@@ -63,6 +60,22 @@ module QPush
|
|
63
60
|
}
|
64
61
|
end
|
65
62
|
|
63
|
+
def delay_namespace
|
64
|
+
@delay_namespace ||= "qpush:v1:#{@namespace}:delay"
|
65
|
+
end
|
66
|
+
|
67
|
+
def queue_namespace
|
68
|
+
@queue_namespace ||= "qpush:v1:#{@namespace}:queue"
|
69
|
+
end
|
70
|
+
|
71
|
+
def perform_namespace
|
72
|
+
@perform_namespace ||= "qpush:v1:#{@namespace}:perform"
|
73
|
+
end
|
74
|
+
|
75
|
+
def stats_namespace
|
76
|
+
@stats_namespace ||= "qpush:v1:#{@namespace}:stats"
|
77
|
+
end
|
78
|
+
|
66
79
|
def perform_lists
|
67
80
|
(1..priorities).collect { |num| "#{perform_namespace}:#{num}" }
|
68
81
|
end
|
data/lib/qpush/job.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
module QPush
|
2
2
|
class << self
|
3
3
|
def job(options)
|
4
|
-
job = Job::
|
4
|
+
job = Job::ClientWrapper.new(options)
|
5
5
|
job.queue
|
6
6
|
end
|
7
7
|
end
|
@@ -19,7 +19,26 @@ module QPush
|
|
19
19
|
end
|
20
20
|
end
|
21
21
|
|
22
|
-
|
22
|
+
class Base
|
23
|
+
attr_accessor :klass, :id, :priority, :created_at, :start_at,
|
24
|
+
:cron, :retry_max, :total_success, :total_fail,
|
25
|
+
:run_time, :namespace
|
26
|
+
attr_reader :args
|
27
|
+
|
28
|
+
def initialize(options = {})
|
29
|
+
options = defaults.merge(options)
|
30
|
+
options.each { |key, value| send("#{key}=", value) }
|
31
|
+
end
|
32
|
+
|
33
|
+
def args=(args)
|
34
|
+
@args =
|
35
|
+
if args.is_a?(String) then JSON.parse(args)
|
36
|
+
else args
|
37
|
+
end
|
38
|
+
rescue JSON::ParserError
|
39
|
+
@args = nil
|
40
|
+
end
|
41
|
+
|
23
42
|
def to_json
|
24
43
|
{ klass: @klass,
|
25
44
|
id: @id,
|
@@ -44,27 +63,16 @@ module QPush
|
|
44
63
|
cron: '',
|
45
64
|
retry_max: 10,
|
46
65
|
total_fail: 0,
|
47
|
-
total_success: 0
|
48
|
-
|
66
|
+
total_success: 0,
|
67
|
+
namespace: QPush.config.namespace }
|
49
68
|
end
|
50
69
|
end
|
51
70
|
|
52
|
-
class
|
53
|
-
include QPush::Job::Base
|
54
|
-
|
55
|
-
attr_accessor :klass, :id, :priority, :created_at, :start_at,
|
56
|
-
:cron, :retry_max, :total_success, :total_fail,
|
57
|
-
:args
|
58
|
-
|
59
|
-
def initialize(options = {})
|
60
|
-
options = defaults.merge(options)
|
61
|
-
options.each { |key, value| send("#{key}=", value) }
|
62
|
-
end
|
63
|
-
|
71
|
+
class ClientWrapper < QPush::Job::Base
|
64
72
|
def queue
|
65
73
|
QPush.redis.with do |conn|
|
66
|
-
conn.incr("
|
67
|
-
conn.lpush("
|
74
|
+
conn.incr("qpush:v1:#{@namespace}:stats:queued")
|
75
|
+
conn.lpush("qpush:v1:#{@namespace}:queue", to_json)
|
68
76
|
end
|
69
77
|
end
|
70
78
|
end
|
data/lib/qpush/server/execute.rb
CHANGED
@@ -39,14 +39,18 @@ module QPush
|
|
39
39
|
end
|
40
40
|
|
41
41
|
def call
|
42
|
-
|
43
|
-
@job.api.retry if @job.retry_job?
|
42
|
+
update_job
|
44
43
|
stat_increment
|
45
44
|
log_error
|
46
45
|
end
|
47
46
|
|
48
47
|
private
|
49
48
|
|
49
|
+
def update_job
|
50
|
+
@job.bump_fail
|
51
|
+
@job.api.retry if @job.retry_job?
|
52
|
+
end
|
53
|
+
|
50
54
|
def stat_increment
|
51
55
|
QPush.redis.with do |c|
|
52
56
|
c.incr("#{QPush.config.stats_namespace}:dead") if @job.dead_job?
|
@@ -70,14 +74,18 @@ module QPush
|
|
70
74
|
end
|
71
75
|
|
72
76
|
def call
|
73
|
-
|
74
|
-
@job.api.delay if @job.delay_job?
|
77
|
+
update_job
|
75
78
|
stat_increment
|
76
79
|
log_success
|
77
80
|
end
|
78
81
|
|
79
82
|
private
|
80
83
|
|
84
|
+
def update_job
|
85
|
+
@job.bump_success
|
86
|
+
@job.api.delay if @job.delay_job?
|
87
|
+
end
|
88
|
+
|
81
89
|
def stat_increment
|
82
90
|
QPush.redis.with do |c|
|
83
91
|
c.incr("#{QPush.config.stats_namespace}:success")
|
data/lib/qpush/server/jobs.rb
CHANGED
@@ -44,30 +44,16 @@ module QPush
|
|
44
44
|
end
|
45
45
|
end
|
46
46
|
|
47
|
-
class Job
|
48
|
-
include QPush::Job::Base
|
47
|
+
class Job < QPush::Job::Base
|
49
48
|
include QPush::Server::JobHelpers
|
50
49
|
include ObjectValidator::Validate
|
51
50
|
|
52
|
-
|
53
|
-
:cron, :retry_max, :total_success, :total_fail,
|
54
|
-
:run_time
|
55
|
-
attr_reader :args, :api
|
51
|
+
attr_reader :api
|
56
52
|
|
57
|
-
def initialize(options
|
58
|
-
|
59
|
-
options.each { |key, value| send("#{key}=", value) }
|
53
|
+
def initialize(options)
|
54
|
+
super
|
60
55
|
@api = JobApi.new(self)
|
61
56
|
end
|
62
|
-
|
63
|
-
def args=(args)
|
64
|
-
@args =
|
65
|
-
if args.is_a?(String) then JSON.parse(args)
|
66
|
-
else args
|
67
|
-
end
|
68
|
-
rescue JSON::ParserError
|
69
|
-
@args = nil
|
70
|
-
end
|
71
57
|
end
|
72
58
|
|
73
59
|
class JobValidator
|
@@ -91,20 +77,12 @@ module QPush
|
|
91
77
|
class JobApi
|
92
78
|
def initialize(job)
|
93
79
|
@job = job
|
94
|
-
@config = QPush.config
|
95
|
-
end
|
96
|
-
|
97
|
-
def delay
|
98
|
-
QPush.redis.with do |conn|
|
99
|
-
conn.incr("#{@config.stats_namespace}:delayed")
|
100
|
-
conn.zadd(@config.delay_namespace, @job.delay_until, @job.to_json)
|
101
|
-
end
|
102
80
|
end
|
103
81
|
|
104
82
|
def queue
|
105
83
|
QPush.redis.with do |conn|
|
106
|
-
conn.incr("#{
|
107
|
-
conn.lpush("#{
|
84
|
+
conn.incr("#{QPush.config.stats_namespace}:queued")
|
85
|
+
conn.lpush("#{QPush.config.queue_namespace}", @job.to_json)
|
108
86
|
end
|
109
87
|
end
|
110
88
|
|
@@ -115,16 +93,17 @@ module QPush
|
|
115
93
|
|
116
94
|
def perform
|
117
95
|
QPush.redis.with do |conn|
|
118
|
-
conn.incr("#{
|
119
|
-
conn.lpush("#{
|
96
|
+
conn.incr("#{QPush.config.stats_namespace}:performed")
|
97
|
+
conn.lpush("#{QPush.config.perform_namespace}:#{@job.priority}", @job.to_json)
|
120
98
|
end
|
121
99
|
end
|
122
100
|
|
101
|
+
def delay
|
102
|
+
send_to_delay('delayed', @job.delay_until)
|
103
|
+
end
|
104
|
+
|
123
105
|
def retry
|
124
|
-
|
125
|
-
conn.incr("#{@config.stats_namespace}:retries")
|
126
|
-
conn.zadd(@config.delay_namespace, @job.retry_at, @job.to_json)
|
127
|
-
end
|
106
|
+
send_to_delay('retries', @job.retry_at)
|
128
107
|
end
|
129
108
|
|
130
109
|
def setup
|
@@ -134,6 +113,15 @@ module QPush
|
|
134
113
|
rescue
|
135
114
|
raise ServerError, 'Invalid job: ' + @job.errors.full_messages.join(' ')
|
136
115
|
end
|
116
|
+
|
117
|
+
private
|
118
|
+
|
119
|
+
def send_to_delay(stat, time)
|
120
|
+
QPush.redis.with do |conn|
|
121
|
+
conn.incr("#{QPush.config.stats_namespace}:#{stat}")
|
122
|
+
conn.zadd(QPush.config.delay_namespace, time, @job.to_json)
|
123
|
+
end
|
124
|
+
end
|
137
125
|
end
|
138
126
|
end
|
139
127
|
end
|
@@ -12,6 +12,7 @@ module QPush
|
|
12
12
|
def start
|
13
13
|
start_message
|
14
14
|
setup_options
|
15
|
+
require_jobs
|
15
16
|
boot_manager
|
16
17
|
end
|
17
18
|
|
@@ -38,6 +39,12 @@ module QPush
|
|
38
39
|
parser.parse!(@argv)
|
39
40
|
end
|
40
41
|
|
42
|
+
def require_jobs
|
43
|
+
Dir[Dir.pwd + "#{QPush.config.jobs_path}/**/*.rb"].each do |file|
|
44
|
+
require file
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
41
48
|
def boot_manager
|
42
49
|
manager = Manager.new(QPush.config.manager_options)
|
43
50
|
manager.start
|
data/lib/qpush/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: qpush
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Nicholas Sweeting
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-07-
|
11
|
+
date: 2016-07-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: redis
|
@@ -171,11 +171,11 @@ files:
|
|
171
171
|
- bin/qpush-server
|
172
172
|
- bin/qpush-web
|
173
173
|
- bin/setup
|
174
|
+
- jobs/test_job.rb
|
174
175
|
- lib/qpush.rb
|
175
176
|
- lib/qpush/base.rb
|
176
177
|
- lib/qpush/config.rb
|
177
178
|
- lib/qpush/job.rb
|
178
|
-
- lib/qpush/jobs/test_job.rb
|
179
179
|
- lib/qpush/redis.rb
|
180
180
|
- lib/qpush/server.rb
|
181
181
|
- lib/qpush/server/database.rb
|