qpush 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.byebug_history +74 -0
- data/bin/qpush-server +0 -1
- data/bin/qpush-web +7 -1
- data/jobs/fail_job.rb +11 -0
- data/jobs/test_job.rb +2 -0
- data/lib/qpush/config.rb +0 -20
- data/lib/qpush/job.rb +11 -7
- data/lib/qpush/jobs/queue_delayed.rb +17 -0
- data/lib/qpush/redis.rb +36 -0
- data/lib/qpush/server.rb +20 -2
- data/lib/qpush/server/apis.rb +46 -0
- data/lib/qpush/server/apis/delay.rb +37 -0
- data/lib/qpush/server/apis/execute.rb +32 -0
- data/lib/qpush/server/apis/fail.rb +39 -0
- data/lib/qpush/server/apis/history.rb +36 -0
- data/lib/qpush/server/apis/morgue.rb +24 -0
- data/lib/qpush/server/apis/perform.rb +24 -0
- data/lib/qpush/server/apis/queue.rb +24 -0
- data/lib/qpush/server/apis/setup.rb +27 -0
- data/lib/qpush/server/apis/success.rb +39 -0
- data/lib/qpush/server/delay.rb +5 -5
- data/lib/qpush/server/errors.rb +1 -16
- data/lib/qpush/server/heartbeat.rb +28 -0
- data/lib/qpush/server/jobs.rb +10 -55
- data/lib/qpush/server/launcher.rb +5 -5
- data/lib/qpush/server/loader.rb +38 -0
- data/lib/qpush/server/perform.rb +2 -2
- data/lib/qpush/server/queue.rb +2 -2
- data/lib/qpush/server/worker.rb +1 -0
- data/lib/qpush/version.rb +1 -1
- data/lib/qpush/web.rb +10 -1
- data/lib/qpush/web/apis/crons.rb +34 -0
- data/lib/qpush/web/apis/heart.rb +12 -0
- data/lib/qpush/web/apis/history.rb +28 -0
- data/lib/qpush/web/apis/jobs.rb +26 -0
- data/lib/qpush/web/apis/morgue.rb +30 -0
- data/lib/qpush/web/apis/queue_delayed.rb +29 -0
- data/lib/qpush/web/apis/queued.rb +0 -0
- data/lib/qpush/web/apis/retries.rb +40 -0
- data/lib/qpush/web/apis/stats.rb +58 -0
- data/lib/qpush/web/get.rb +32 -34
- data/lib/qpush/web/post.rb +8 -0
- data/lib/qpush/web/server.rb +34 -5
- data/qpush.gemspec +1 -0
- metadata +40 -3
- data/lib/qpush/server/execute.rb +0 -100
@@ -0,0 +1,36 @@
|
|
1
|
+
module QPush
|
2
|
+
module Server
|
3
|
+
module Apis
|
4
|
+
class History < Base
|
5
|
+
def initialize(job, status, error)
|
6
|
+
@status = status
|
7
|
+
@klass = job.klass
|
8
|
+
@args = job.args
|
9
|
+
@performed = Time.now.to_i
|
10
|
+
@error = error ? error.message : nil
|
11
|
+
end
|
12
|
+
|
13
|
+
def call
|
14
|
+
update_history
|
15
|
+
end
|
16
|
+
|
17
|
+
private
|
18
|
+
|
19
|
+
def update_history
|
20
|
+
QPush.redis.with do |c|
|
21
|
+
c.lpush(QPush.keys.history, to_json)
|
22
|
+
c.ltrim(QPush.keys.history, 0, 10)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def to_json
|
27
|
+
{ status: @status,
|
28
|
+
klass: @klass,
|
29
|
+
args: @args,
|
30
|
+
performed: @performed,
|
31
|
+
error: @error }.to_json
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module QPush
|
2
|
+
module Server
|
3
|
+
module Apis
|
4
|
+
class Morgue < Base
|
5
|
+
def initialize(job)
|
6
|
+
@job = job
|
7
|
+
end
|
8
|
+
|
9
|
+
def call
|
10
|
+
send_to_morgue
|
11
|
+
end
|
12
|
+
|
13
|
+
private
|
14
|
+
|
15
|
+
def send_to_morgue
|
16
|
+
QPush.redis.with do |conn|
|
17
|
+
conn.hincrby(QPush.keys.stats, 'dead', 1)
|
18
|
+
conn.lpush(QPush.keys.morgue, @job.to_json)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module QPush
|
2
|
+
module Server
|
3
|
+
module Apis
|
4
|
+
class Perform < Base
|
5
|
+
def initialize(job)
|
6
|
+
@job = job
|
7
|
+
end
|
8
|
+
|
9
|
+
def call
|
10
|
+
perform_job
|
11
|
+
end
|
12
|
+
|
13
|
+
private
|
14
|
+
|
15
|
+
def perform_job
|
16
|
+
QPush.redis.with do |conn|
|
17
|
+
conn.hincrby(QPush.keys.stats, 'performed', 1)
|
18
|
+
conn.lpush("#{QPush.keys.perform}:#{@job.priority}", @job.to_json)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module QPush
|
2
|
+
module Server
|
3
|
+
module Apis
|
4
|
+
class Queue < Base
|
5
|
+
def initialize(job)
|
6
|
+
@job = job
|
7
|
+
end
|
8
|
+
|
9
|
+
def call
|
10
|
+
queue_job
|
11
|
+
end
|
12
|
+
|
13
|
+
private
|
14
|
+
|
15
|
+
def queue_job
|
16
|
+
QPush.redis.with do |conn|
|
17
|
+
conn.hincrby(QPush.keys.stats, 'queued', 1)
|
18
|
+
conn.lpush(QPush.keys.queue, @job.to_json)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
module QPush
|
2
|
+
module Server
|
3
|
+
module Apis
|
4
|
+
class Setup < Base
|
5
|
+
def initialize(job)
|
6
|
+
@job = job
|
7
|
+
end
|
8
|
+
|
9
|
+
def call
|
10
|
+
invalid_job && return unless @job.valid?
|
11
|
+
setup_job
|
12
|
+
end
|
13
|
+
|
14
|
+
private
|
15
|
+
|
16
|
+
def setup_job
|
17
|
+
Perform.call(@job) if @job.perform_job?
|
18
|
+
Delay.call(@job, :delay) if @job.delay_job?
|
19
|
+
end
|
20
|
+
|
21
|
+
def invalid_job
|
22
|
+
Server.log.err("Job INVALID | #{@job.klass} | #{@job.id} | #{@job.errors.full_messages.join(' ')}")
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
module QPush
|
2
|
+
module Server
|
3
|
+
module Apis
|
4
|
+
class Success < Base
|
5
|
+
def initialize(job)
|
6
|
+
@job = job
|
7
|
+
end
|
8
|
+
|
9
|
+
def call
|
10
|
+
update_job
|
11
|
+
stat_increment
|
12
|
+
log_success
|
13
|
+
update_history
|
14
|
+
end
|
15
|
+
|
16
|
+
private
|
17
|
+
|
18
|
+
def update_job
|
19
|
+
@job.mark_success
|
20
|
+
@job.delay if @job.delay_job?
|
21
|
+
end
|
22
|
+
|
23
|
+
def stat_increment
|
24
|
+
QPush.redis.with do |c|
|
25
|
+
c.hincrby(QPush.keys.stats, 'success', 1)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def log_success
|
30
|
+
Server.log.info("Job SUCCESS | #{@job.klass} with ID: #{@job.id} | #{@job.run_time}")
|
31
|
+
end
|
32
|
+
|
33
|
+
def update_history
|
34
|
+
History.call(@job, true, nil)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
data/lib/qpush/server/delay.rb
CHANGED
@@ -22,7 +22,7 @@ module QPush
|
|
22
22
|
end
|
23
23
|
end
|
24
24
|
|
25
|
-
# Shutsdown our
|
25
|
+
# Shutsdown our delay process.
|
26
26
|
#
|
27
27
|
def shutdown
|
28
28
|
@done = true
|
@@ -34,7 +34,7 @@ module QPush
|
|
34
34
|
# If any are found, begin to update them.
|
35
35
|
#
|
36
36
|
def retrieve_delays
|
37
|
-
delays = @conn.zrangebyscore(QPush.
|
37
|
+
delays = @conn.zrangebyscore(QPush.keys.delay, 0, Time.now.to_i)
|
38
38
|
delays.any? ? update_delays(delays) : @conn.unwatch
|
39
39
|
end
|
40
40
|
|
@@ -42,7 +42,7 @@ module QPush
|
|
42
42
|
#
|
43
43
|
def update_delays(delays)
|
44
44
|
@conn.multi do |multi|
|
45
|
-
multi.zrem(QPush.
|
45
|
+
multi.zrem(QPush.keys.delay, delays)
|
46
46
|
delays.each { |job| perform_job(job) }
|
47
47
|
end
|
48
48
|
end
|
@@ -51,7 +51,7 @@ module QPush
|
|
51
51
|
#
|
52
52
|
def perform_job(json)
|
53
53
|
job = Job.new(JSON.parse(json))
|
54
|
-
job.
|
54
|
+
job.perform
|
55
55
|
rescue => e
|
56
56
|
raise ServerError, e.message
|
57
57
|
end
|
@@ -59,7 +59,7 @@ module QPush
|
|
59
59
|
# Performs a watch on our delay list
|
60
60
|
#
|
61
61
|
def watch_delay
|
62
|
-
@conn.watch(QPush.
|
62
|
+
@conn.watch(QPush.keys.delay) do
|
63
63
|
yield if block_given?
|
64
64
|
end
|
65
65
|
end
|
data/lib/qpush/server/errors.rb
CHANGED
@@ -1,18 +1,3 @@
|
|
1
1
|
module QPush
|
2
|
-
class ServerError < StandardError
|
3
|
-
def initialize(msg = nil)
|
4
|
-
@message = msg
|
5
|
-
log_error
|
6
|
-
end
|
7
|
-
|
8
|
-
def message
|
9
|
-
"The following error occured: #{@message}"
|
10
|
-
end
|
11
|
-
|
12
|
-
private
|
13
|
-
|
14
|
-
def log_error
|
15
|
-
Server.log.err("Server Error - #{@message}")
|
16
|
-
end
|
17
|
-
end
|
2
|
+
class ServerError < StandardError; end
|
18
3
|
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module QPush
|
2
|
+
module Server
|
3
|
+
# The Heartbeat worker periodically updates the heart namespace.
|
4
|
+
# The key is set with an expiry. This helps to indicate if the QPush server
|
5
|
+
# is currently active.
|
6
|
+
#
|
7
|
+
class Heartbeat
|
8
|
+
def initialize
|
9
|
+
@done = false
|
10
|
+
end
|
11
|
+
|
12
|
+
# Starts our perform process. This will run until instructed to stop.
|
13
|
+
#
|
14
|
+
def start
|
15
|
+
until @done
|
16
|
+
QPush.redis.with { |c| c.setex(QPush.keys.heart, 30, true) }
|
17
|
+
sleep 15
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
# Shutsdown our heartbeat process.
|
22
|
+
#
|
23
|
+
def shutdown
|
24
|
+
@done = true
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
data/lib/qpush/server/jobs.rb
CHANGED
@@ -1,11 +1,13 @@
|
|
1
1
|
module QPush
|
2
2
|
module Server
|
3
3
|
module JobHelpers
|
4
|
-
def
|
4
|
+
def mark_success
|
5
|
+
@failed = false
|
5
6
|
@total_success += 1
|
6
7
|
end
|
7
8
|
|
8
|
-
def
|
9
|
+
def mark_failed
|
10
|
+
@failed = true
|
9
11
|
@total_fail += 1
|
10
12
|
end
|
11
13
|
|
@@ -45,15 +47,18 @@ module QPush
|
|
45
47
|
end
|
46
48
|
|
47
49
|
class Job < QPush::Job::Base
|
50
|
+
extend Forwardable
|
51
|
+
|
48
52
|
include QPush::Server::JobHelpers
|
49
53
|
include ObjectValidator::Validate
|
50
54
|
|
51
|
-
attr_reader :api
|
52
|
-
|
53
55
|
def initialize(options)
|
54
56
|
super
|
55
|
-
@api =
|
57
|
+
@api = ApiWrapper.new(self)
|
56
58
|
end
|
59
|
+
|
60
|
+
def_delegators :@api, :queue, :execute, :perform,
|
61
|
+
:delay, :retry, :morgue, :setup
|
57
62
|
end
|
58
63
|
|
59
64
|
class JobValidator
|
@@ -73,55 +78,5 @@ module QPush
|
|
73
78
|
validates :total_fail, type: Integer
|
74
79
|
validates :total_success, type: Integer
|
75
80
|
end
|
76
|
-
|
77
|
-
class JobApi
|
78
|
-
def initialize(job)
|
79
|
-
@job = job
|
80
|
-
end
|
81
|
-
|
82
|
-
def queue
|
83
|
-
QPush.redis.with do |conn|
|
84
|
-
conn.incr("#{QPush.config.stats_namespace}:queued")
|
85
|
-
conn.lpush("#{QPush.config.queue_namespace}", @job.to_json)
|
86
|
-
end
|
87
|
-
end
|
88
|
-
|
89
|
-
def execute
|
90
|
-
execute = Execute.new(@job)
|
91
|
-
execute.call
|
92
|
-
end
|
93
|
-
|
94
|
-
def perform
|
95
|
-
QPush.redis.with do |conn|
|
96
|
-
conn.incr("#{QPush.config.stats_namespace}:performed")
|
97
|
-
conn.lpush("#{QPush.config.perform_namespace}:#{@job.priority}", @job.to_json)
|
98
|
-
end
|
99
|
-
end
|
100
|
-
|
101
|
-
def delay
|
102
|
-
send_to_delay('delayed', @job.delay_until)
|
103
|
-
end
|
104
|
-
|
105
|
-
def retry
|
106
|
-
send_to_delay('retries', @job.retry_at)
|
107
|
-
end
|
108
|
-
|
109
|
-
def setup
|
110
|
-
fail unless @job.valid?
|
111
|
-
perform if @job.perform_job?
|
112
|
-
delay if @job.delay_job?
|
113
|
-
rescue
|
114
|
-
raise ServerError, 'Invalid job: ' + @job.errors.full_messages.join(' ')
|
115
|
-
end
|
116
|
-
|
117
|
-
private
|
118
|
-
|
119
|
-
def send_to_delay(stat, time)
|
120
|
-
QPush.redis.with do |conn|
|
121
|
-
conn.incr("#{QPush.config.stats_namespace}:#{stat}")
|
122
|
-
conn.zadd(QPush.config.delay_namespace, time, @job.to_json)
|
123
|
-
end
|
124
|
-
end
|
125
|
-
end
|
126
81
|
end
|
127
82
|
end
|
@@ -12,7 +12,7 @@ module QPush
|
|
12
12
|
def start
|
13
13
|
start_message
|
14
14
|
setup_options
|
15
|
-
|
15
|
+
setup_jobs
|
16
16
|
boot_manager
|
17
17
|
end
|
18
18
|
|
@@ -39,10 +39,10 @@ module QPush
|
|
39
39
|
parser.parse!(@argv)
|
40
40
|
end
|
41
41
|
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
42
|
+
# Requires all base jobs as well as user jobs.
|
43
|
+
#
|
44
|
+
def setup_jobs
|
45
|
+
Loader.call
|
46
46
|
end
|
47
47
|
|
48
48
|
def boot_manager
|
@@ -0,0 +1,38 @@
|
|
1
|
+
module QPush
|
2
|
+
module Server
|
3
|
+
# The Loader will 'require' all jobs within the users job folder.
|
4
|
+
# The job folder is specified in the config.
|
5
|
+
#
|
6
|
+
class Loader
|
7
|
+
# Provides a shortend caller.
|
8
|
+
#
|
9
|
+
def self.call
|
10
|
+
jobs = Loader.new
|
11
|
+
jobs.call
|
12
|
+
end
|
13
|
+
|
14
|
+
# Entrypoint to load all jobs.
|
15
|
+
#
|
16
|
+
def call
|
17
|
+
remove_old
|
18
|
+
load_jobs
|
19
|
+
end
|
20
|
+
|
21
|
+
private
|
22
|
+
|
23
|
+
# Removes old jobs from the redis job list.
|
24
|
+
#
|
25
|
+
def remove_old
|
26
|
+
QPush.redis.with { |c| c.del(QPush.keys.jobs) }
|
27
|
+
end
|
28
|
+
|
29
|
+
# Requires user jobs that are specified from the config.
|
30
|
+
#
|
31
|
+
def load_jobs
|
32
|
+
Dir[Dir.pwd + "#{QPush.config.jobs_path}/**/*.rb"].each do |file|
|
33
|
+
require file
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
data/lib/qpush/server/perform.rb
CHANGED
@@ -6,7 +6,7 @@ module QPush
|
|
6
6
|
class Perform
|
7
7
|
def initialize
|
8
8
|
@done = false
|
9
|
-
@lists = QPush.
|
9
|
+
@lists = QPush.keys.perform_lists
|
10
10
|
end
|
11
11
|
|
12
12
|
# Starts our perform process. This will run until instructed to stop.
|
@@ -14,7 +14,7 @@ module QPush
|
|
14
14
|
def start
|
15
15
|
until @done
|
16
16
|
job = retrieve_job
|
17
|
-
job.
|
17
|
+
job.execute if job
|
18
18
|
end
|
19
19
|
end
|
20
20
|
|