qpush 0.1.2 → 0.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +4 -4
  2. data/.byebug_history +74 -0
  3. data/bin/qpush-server +0 -1
  4. data/bin/qpush-web +7 -1
  5. data/jobs/fail_job.rb +11 -0
  6. data/jobs/test_job.rb +2 -0
  7. data/lib/qpush/config.rb +0 -20
  8. data/lib/qpush/job.rb +11 -7
  9. data/lib/qpush/jobs/queue_delayed.rb +17 -0
  10. data/lib/qpush/redis.rb +36 -0
  11. data/lib/qpush/server.rb +20 -2
  12. data/lib/qpush/server/apis.rb +46 -0
  13. data/lib/qpush/server/apis/delay.rb +37 -0
  14. data/lib/qpush/server/apis/execute.rb +32 -0
  15. data/lib/qpush/server/apis/fail.rb +39 -0
  16. data/lib/qpush/server/apis/history.rb +36 -0
  17. data/lib/qpush/server/apis/morgue.rb +24 -0
  18. data/lib/qpush/server/apis/perform.rb +24 -0
  19. data/lib/qpush/server/apis/queue.rb +24 -0
  20. data/lib/qpush/server/apis/setup.rb +27 -0
  21. data/lib/qpush/server/apis/success.rb +39 -0
  22. data/lib/qpush/server/delay.rb +5 -5
  23. data/lib/qpush/server/errors.rb +1 -16
  24. data/lib/qpush/server/heartbeat.rb +28 -0
  25. data/lib/qpush/server/jobs.rb +10 -55
  26. data/lib/qpush/server/launcher.rb +5 -5
  27. data/lib/qpush/server/loader.rb +38 -0
  28. data/lib/qpush/server/perform.rb +2 -2
  29. data/lib/qpush/server/queue.rb +2 -2
  30. data/lib/qpush/server/worker.rb +1 -0
  31. data/lib/qpush/version.rb +1 -1
  32. data/lib/qpush/web.rb +10 -1
  33. data/lib/qpush/web/apis/crons.rb +34 -0
  34. data/lib/qpush/web/apis/heart.rb +12 -0
  35. data/lib/qpush/web/apis/history.rb +28 -0
  36. data/lib/qpush/web/apis/jobs.rb +26 -0
  37. data/lib/qpush/web/apis/morgue.rb +30 -0
  38. data/lib/qpush/web/apis/queue_delayed.rb +29 -0
  39. data/lib/qpush/web/apis/queued.rb +0 -0
  40. data/lib/qpush/web/apis/retries.rb +40 -0
  41. data/lib/qpush/web/apis/stats.rb +58 -0
  42. data/lib/qpush/web/get.rb +32 -34
  43. data/lib/qpush/web/post.rb +8 -0
  44. data/lib/qpush/web/server.rb +34 -5
  45. data/qpush.gemspec +1 -0
  46. metadata +40 -3
  47. data/lib/qpush/server/execute.rb +0 -100
@@ -0,0 +1,36 @@
1
+ module QPush
2
+ module Server
3
+ module Apis
4
+ class History < Base
5
+ def initialize(job, status, error)
6
+ @status = status
7
+ @klass = job.klass
8
+ @args = job.args
9
+ @performed = Time.now.to_i
10
+ @error = error ? error.message : nil
11
+ end
12
+
13
+ def call
14
+ update_history
15
+ end
16
+
17
+ private
18
+
19
+ def update_history
20
+ QPush.redis.with do |c|
21
+ c.lpush(QPush.keys.history, to_json)
22
+ c.ltrim(QPush.keys.history, 0, 10)
23
+ end
24
+ end
25
+
26
+ def to_json
27
+ { status: @status,
28
+ klass: @klass,
29
+ args: @args,
30
+ performed: @performed,
31
+ error: @error }.to_json
32
+ end
33
+ end
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,24 @@
1
+ module QPush
2
+ module Server
3
+ module Apis
4
+ class Morgue < Base
5
+ def initialize(job)
6
+ @job = job
7
+ end
8
+
9
+ def call
10
+ send_to_morgue
11
+ end
12
+
13
+ private
14
+
15
+ def send_to_morgue
16
+ QPush.redis.with do |conn|
17
+ conn.hincrby(QPush.keys.stats, 'dead', 1)
18
+ conn.lpush(QPush.keys.morgue, @job.to_json)
19
+ end
20
+ end
21
+ end
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,24 @@
1
+ module QPush
2
+ module Server
3
+ module Apis
4
+ class Perform < Base
5
+ def initialize(job)
6
+ @job = job
7
+ end
8
+
9
+ def call
10
+ perform_job
11
+ end
12
+
13
+ private
14
+
15
+ def perform_job
16
+ QPush.redis.with do |conn|
17
+ conn.hincrby(QPush.keys.stats, 'performed', 1)
18
+ conn.lpush("#{QPush.keys.perform}:#{@job.priority}", @job.to_json)
19
+ end
20
+ end
21
+ end
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,24 @@
1
+ module QPush
2
+ module Server
3
+ module Apis
4
+ class Queue < Base
5
+ def initialize(job)
6
+ @job = job
7
+ end
8
+
9
+ def call
10
+ queue_job
11
+ end
12
+
13
+ private
14
+
15
+ def queue_job
16
+ QPush.redis.with do |conn|
17
+ conn.hincrby(QPush.keys.stats, 'queued', 1)
18
+ conn.lpush(QPush.keys.queue, @job.to_json)
19
+ end
20
+ end
21
+ end
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,27 @@
1
+ module QPush
2
+ module Server
3
+ module Apis
4
+ class Setup < Base
5
+ def initialize(job)
6
+ @job = job
7
+ end
8
+
9
+ def call
10
+ invalid_job && return unless @job.valid?
11
+ setup_job
12
+ end
13
+
14
+ private
15
+
16
+ def setup_job
17
+ Perform.call(@job) if @job.perform_job?
18
+ Delay.call(@job, :delay) if @job.delay_job?
19
+ end
20
+
21
+ def invalid_job
22
+ Server.log.err("Job INVALID | #{@job.klass} | #{@job.id} | #{@job.errors.full_messages.join(' ')}")
23
+ end
24
+ end
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,39 @@
1
+ module QPush
2
+ module Server
3
+ module Apis
4
+ class Success < Base
5
+ def initialize(job)
6
+ @job = job
7
+ end
8
+
9
+ def call
10
+ update_job
11
+ stat_increment
12
+ log_success
13
+ update_history
14
+ end
15
+
16
+ private
17
+
18
+ def update_job
19
+ @job.mark_success
20
+ @job.delay if @job.delay_job?
21
+ end
22
+
23
+ def stat_increment
24
+ QPush.redis.with do |c|
25
+ c.hincrby(QPush.keys.stats, 'success', 1)
26
+ end
27
+ end
28
+
29
+ def log_success
30
+ Server.log.info("Job SUCCESS | #{@job.klass} with ID: #{@job.id} | #{@job.run_time}")
31
+ end
32
+
33
+ def update_history
34
+ History.call(@job, true, nil)
35
+ end
36
+ end
37
+ end
38
+ end
39
+ end
@@ -22,7 +22,7 @@ module QPush
22
22
  end
23
23
  end
24
24
 
25
- # Shutsdown our dleay process.
25
+ # Shutsdown our delay process.
26
26
  #
27
27
  def shutdown
28
28
  @done = true
@@ -34,7 +34,7 @@ module QPush
34
34
  # If any are found, begin to update them.
35
35
  #
36
36
  def retrieve_delays
37
- delays = @conn.zrangebyscore(QPush.config.delay_namespace, 0, Time.now.to_i)
37
+ delays = @conn.zrangebyscore(QPush.keys.delay, 0, Time.now.to_i)
38
38
  delays.any? ? update_delays(delays) : @conn.unwatch
39
39
  end
40
40
 
@@ -42,7 +42,7 @@ module QPush
42
42
  #
43
43
  def update_delays(delays)
44
44
  @conn.multi do |multi|
45
- multi.zrem(QPush.config.delay_namespace, delays)
45
+ multi.zrem(QPush.keys.delay, delays)
46
46
  delays.each { |job| perform_job(job) }
47
47
  end
48
48
  end
@@ -51,7 +51,7 @@ module QPush
51
51
  #
52
52
  def perform_job(json)
53
53
  job = Job.new(JSON.parse(json))
54
- job.api.perform
54
+ job.perform
55
55
  rescue => e
56
56
  raise ServerError, e.message
57
57
  end
@@ -59,7 +59,7 @@ module QPush
59
59
  # Performs a watch on our delay list
60
60
  #
61
61
  def watch_delay
62
- @conn.watch(QPush.config.delay_namespace) do
62
+ @conn.watch(QPush.keys.delay) do
63
63
  yield if block_given?
64
64
  end
65
65
  end
@@ -1,18 +1,3 @@
1
1
  module QPush
2
- class ServerError < StandardError
3
- def initialize(msg = nil)
4
- @message = msg
5
- log_error
6
- end
7
-
8
- def message
9
- "The following error occured: #{@message}"
10
- end
11
-
12
- private
13
-
14
- def log_error
15
- Server.log.err("Server Error - #{@message}")
16
- end
17
- end
2
+ class ServerError < StandardError; end
18
3
  end
@@ -0,0 +1,28 @@
1
+ module QPush
2
+ module Server
3
+ # The Heartbeat worker periodically updates the heart namespace.
4
+ # The key is set with an expiry. This helps to indicate if the QPush server
5
+ # is currently active.
6
+ #
7
+ class Heartbeat
8
+ def initialize
9
+ @done = false
10
+ end
11
+
12
+ # Starts our perform process. This will run until instructed to stop.
13
+ #
14
+ def start
15
+ until @done
16
+ QPush.redis.with { |c| c.setex(QPush.keys.heart, 30, true) }
17
+ sleep 15
18
+ end
19
+ end
20
+
21
+ # Shutsdown our heartbeat process.
22
+ #
23
+ def shutdown
24
+ @done = true
25
+ end
26
+ end
27
+ end
28
+ end
@@ -1,11 +1,13 @@
1
1
  module QPush
2
2
  module Server
3
3
  module JobHelpers
4
- def bump_success
4
+ def mark_success
5
+ @failed = false
5
6
  @total_success += 1
6
7
  end
7
8
 
8
- def bump_fail
9
+ def mark_failed
10
+ @failed = true
9
11
  @total_fail += 1
10
12
  end
11
13
 
@@ -45,15 +47,18 @@ module QPush
45
47
  end
46
48
 
47
49
  class Job < QPush::Job::Base
50
+ extend Forwardable
51
+
48
52
  include QPush::Server::JobHelpers
49
53
  include ObjectValidator::Validate
50
54
 
51
- attr_reader :api
52
-
53
55
  def initialize(options)
54
56
  super
55
- @api = JobApi.new(self)
57
+ @api = ApiWrapper.new(self)
56
58
  end
59
+
60
+ def_delegators :@api, :queue, :execute, :perform,
61
+ :delay, :retry, :morgue, :setup
57
62
  end
58
63
 
59
64
  class JobValidator
@@ -73,55 +78,5 @@ module QPush
73
78
  validates :total_fail, type: Integer
74
79
  validates :total_success, type: Integer
75
80
  end
76
-
77
- class JobApi
78
- def initialize(job)
79
- @job = job
80
- end
81
-
82
- def queue
83
- QPush.redis.with do |conn|
84
- conn.incr("#{QPush.config.stats_namespace}:queued")
85
- conn.lpush("#{QPush.config.queue_namespace}", @job.to_json)
86
- end
87
- end
88
-
89
- def execute
90
- execute = Execute.new(@job)
91
- execute.call
92
- end
93
-
94
- def perform
95
- QPush.redis.with do |conn|
96
- conn.incr("#{QPush.config.stats_namespace}:performed")
97
- conn.lpush("#{QPush.config.perform_namespace}:#{@job.priority}", @job.to_json)
98
- end
99
- end
100
-
101
- def delay
102
- send_to_delay('delayed', @job.delay_until)
103
- end
104
-
105
- def retry
106
- send_to_delay('retries', @job.retry_at)
107
- end
108
-
109
- def setup
110
- fail unless @job.valid?
111
- perform if @job.perform_job?
112
- delay if @job.delay_job?
113
- rescue
114
- raise ServerError, 'Invalid job: ' + @job.errors.full_messages.join(' ')
115
- end
116
-
117
- private
118
-
119
- def send_to_delay(stat, time)
120
- QPush.redis.with do |conn|
121
- conn.incr("#{QPush.config.stats_namespace}:#{stat}")
122
- conn.zadd(QPush.config.delay_namespace, time, @job.to_json)
123
- end
124
- end
125
- end
126
81
  end
127
82
  end
@@ -12,7 +12,7 @@ module QPush
12
12
  def start
13
13
  start_message
14
14
  setup_options
15
- require_jobs
15
+ setup_jobs
16
16
  boot_manager
17
17
  end
18
18
 
@@ -39,10 +39,10 @@ module QPush
39
39
  parser.parse!(@argv)
40
40
  end
41
41
 
42
- def require_jobs
43
- Dir[Dir.pwd + "#{QPush.config.jobs_path}/**/*.rb"].each do |file|
44
- require file
45
- end
42
+ # Requires all base jobs as well as user jobs.
43
+ #
44
+ def setup_jobs
45
+ Loader.call
46
46
  end
47
47
 
48
48
  def boot_manager
@@ -0,0 +1,38 @@
1
+ module QPush
2
+ module Server
3
+ # The Loader will 'require' all jobs within the users job folder.
4
+ # The job folder is specified in the config.
5
+ #
6
+ class Loader
7
+ # Provides a shortend caller.
8
+ #
9
+ def self.call
10
+ jobs = Loader.new
11
+ jobs.call
12
+ end
13
+
14
+ # Entrypoint to load all jobs.
15
+ #
16
+ def call
17
+ remove_old
18
+ load_jobs
19
+ end
20
+
21
+ private
22
+
23
+ # Removes old jobs from the redis job list.
24
+ #
25
+ def remove_old
26
+ QPush.redis.with { |c| c.del(QPush.keys.jobs) }
27
+ end
28
+
29
+ # Requires user jobs that are specified from the config.
30
+ #
31
+ def load_jobs
32
+ Dir[Dir.pwd + "#{QPush.config.jobs_path}/**/*.rb"].each do |file|
33
+ require file
34
+ end
35
+ end
36
+ end
37
+ end
38
+ end
@@ -6,7 +6,7 @@ module QPush
6
6
  class Perform
7
7
  def initialize
8
8
  @done = false
9
- @lists = QPush.config.perform_lists
9
+ @lists = QPush.keys.perform_lists
10
10
  end
11
11
 
12
12
  # Starts our perform process. This will run until instructed to stop.
@@ -14,7 +14,7 @@ module QPush
14
14
  def start
15
15
  until @done
16
16
  job = retrieve_job
17
- job.api.execute if job
17
+ job.execute if job
18
18
  end
19
19
  end
20
20