que 0.0.1 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,20 @@
1
+ module Que
2
+ class Railtie < Rails::Railtie
3
+ config.que = Que
4
+ config.que.connection = ::ActiveRecord if defined?(::ActiveRecord)
5
+ config.que.mode = :sync if Rails.env.test?
6
+
7
+ rake_tasks do
8
+ load 'que/rake_tasks.rb'
9
+ end
10
+
11
+ initializer "que.setup" do
12
+ ActiveSupport.on_load(:after_initialize) do
13
+ Que.logger ||= Rails.logger
14
+
15
+ # Only start up the worker pool if running as a server.
16
+ Que.mode ||= :async if defined? Rails::Server
17
+ end
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,35 @@
1
+ require 'logger'
2
+
3
+ namespace :que do
4
+ desc "Process Que's jobs using a worker pool"
5
+ task :work => :environment do
6
+ Que.logger = Logger.new(STDOUT)
7
+ Que.mode = :async
8
+ Que.worker_count = (ENV['WORKER_COUNT'] || 4).to_i
9
+
10
+ %w(INT TERM).each do |signal|
11
+ trap signal do
12
+ puts "SIG#{signal} caught, finishing current jobs and shutting down..."
13
+ Que.mode = :off
14
+ $stop = true
15
+ end
16
+ end
17
+
18
+ loop { sleep 0.01; break if $stop }
19
+ end
20
+
21
+ desc "Create Que's job table"
22
+ task :create => :environment do
23
+ Que.create!
24
+ end
25
+
26
+ desc "Drop Que's job table"
27
+ task :drop => :environment do
28
+ Que.drop!
29
+ end
30
+
31
+ desc "Clear Que's job table"
32
+ task :clear => :environment do
33
+ Que.clear!
34
+ end
35
+ end
data/lib/que/sql.rb ADDED
@@ -0,0 +1,121 @@
1
+ module Que
2
+ SQL = {
3
+ # Thanks to RhodiumToad in #postgresql for the job lock CTE and its lateral
4
+ # variant. They were modified only slightly from his design.
5
+ :lock_job => (
6
+ <<-SQL
7
+ WITH RECURSIVE cte AS (
8
+ SELECT (job).*, pg_try_advisory_lock((job).job_id) AS locked
9
+ FROM (
10
+ SELECT job
11
+ FROM que_jobs AS job
12
+ WHERE run_at <= now()
13
+ ORDER BY priority, run_at, job_id
14
+ LIMIT 1
15
+ ) AS t1
16
+ UNION ALL (
17
+ SELECT (job).*, pg_try_advisory_lock((job).job_id) AS locked
18
+ FROM (
19
+ SELECT (
20
+ SELECT job
21
+ FROM que_jobs AS job
22
+ WHERE run_at <= now() AND (priority, run_at, job_id) > (cte.priority, cte.run_at, cte.job_id)
23
+ ORDER BY priority, run_at, job_id
24
+ LIMIT 1
25
+ ) AS job
26
+ FROM cte
27
+ WHERE NOT cte.locked
28
+ LIMIT 1
29
+ ) AS t1
30
+ )
31
+ )
32
+ SELECT job_id, priority, run_at, args, job_class, error_count
33
+ FROM cte
34
+ WHERE locked
35
+ LIMIT 1
36
+ SQL
37
+ ).freeze,
38
+
39
+ # Here's an alternate scheme using LATERAL, which will work in Postgres 9.3+.
40
+ # Basically the same, but benchmark to see if it's faster/just as reliable.
41
+
42
+ # WITH RECURSIVE cte AS (
43
+ # SELECT *, pg_try_advisory_lock(s.job_id) AS locked
44
+ # FROM (
45
+ # SELECT *
46
+ # FROM que_jobs
47
+ # WHERE run_at <= now()
48
+ # ORDER BY priority, run_at, job_id
49
+ # LIMIT 1
50
+ # ) s
51
+ # UNION ALL (
52
+ # SELECT j.*, pg_try_advisory_lock(j.job_id) AS locked
53
+ # FROM (
54
+ # SELECT *
55
+ # FROM cte
56
+ # WHERE NOT locked
57
+ # ) t,
58
+ # LATERAL (
59
+ # SELECT *
60
+ # FROM que_jobs
61
+ # WHERE run_at <= now()
62
+ # AND (priority, run_at, job_id) > (t.priority, t.run_at, t.job_id)
63
+ # ORDER BY priority, run_at, job_id
64
+ # LIMIT 1
65
+ # ) j
66
+ # )
67
+ # )
68
+ # SELECT *
69
+ # FROM cte
70
+ # WHERE locked
71
+ # LIMIT 1
72
+
73
+ :create_table => (
74
+ <<-SQL
75
+ CREATE TABLE que_jobs
76
+ (
77
+ priority integer NOT NULL DEFAULT 1,
78
+ run_at timestamptz NOT NULL DEFAULT now(),
79
+ job_id bigserial NOT NULL,
80
+ job_class text NOT NULL,
81
+ args json NOT NULL DEFAULT '[]'::json,
82
+ error_count integer NOT NULL DEFAULT 0,
83
+ last_error text,
84
+
85
+ CONSTRAINT que_jobs_pkey PRIMARY KEY (priority, run_at, job_id)
86
+ )
87
+ SQL
88
+ ).freeze,
89
+
90
+ :check_job => (
91
+ <<-SQL
92
+ SELECT 1 AS one
93
+ FROM que_jobs
94
+ WHERE priority = $1::integer
95
+ AND run_at = $2::timestamptz
96
+ AND job_id = $3::bigint
97
+ SQL
98
+ ).freeze,
99
+
100
+ :set_error => (
101
+ <<-SQL
102
+ UPDATE que_jobs
103
+ SET error_count = $1::integer,
104
+ run_at = $2::timestamptz,
105
+ last_error = $3::text
106
+ WHERE priority = $4::integer
107
+ AND run_at = $5::timestamptz
108
+ AND job_id = $6::bigint
109
+ SQL
110
+ ).freeze,
111
+
112
+ :destroy_job => (
113
+ <<-SQL
114
+ DELETE FROM que_jobs
115
+ WHERE priority = $1::integer
116
+ AND run_at = $2::timestamptz
117
+ AND job_id = $3::bigint
118
+ SQL
119
+ ).freeze
120
+ }
121
+ end
data/lib/que/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Que
2
- VERSION = '0.0.1'
2
+ Version = '0.1.0'
3
3
  end
data/lib/que/worker.rb CHANGED
@@ -1,215 +1,152 @@
1
- module Que
2
- # There are multiple workers running at a given time, each with a given
3
- # minimum priority that they care about for jobs. A worker will continuously
4
- # look for jobs and work them. If there's no job available, the worker will go
5
- # to sleep until it is awakened by an external thread.
6
-
7
- # Use Worker.state = ... to set the current state. There are three states:
8
- # :async => Work jobs in dedicated threads. Used in production.
9
- # :sync => Work jobs immediately, as they're queued, in the current thread. Used in testing.
10
- # :off => Don't work jobs at all. Must use Job#work or Job.work explicitly.
11
-
12
- # Worker.wake! will wake up the sleeping worker with the lowest minimum job
13
- # priority. Worker.wake! may be run by another thread handling a web request,
14
- # or by the wrangler thread (which wakes a worker every five seconds, to
15
- # handle scheduled jobs). It only has an effect when running in async mode.
1
+ require 'monitor'
16
2
 
3
+ module Que
17
4
  class Worker
18
- # Each worker has a corresponding thread, which contains two variables:
19
- # :directive, to define what it should be doing, and :state, to define what
20
- # it's actually doing at the moment. Need to be careful that these variables
21
- # are only modified by a single thread at a time (hence, MonitorMixin).
22
- include MonitorMixin
23
-
24
- # The Worker class itself needs to be protected as well, to make sure that
25
- # multiple threads aren't stopping/starting it at the same time.
26
- extend MonitorMixin
5
+ # Each worker has a thread that does the actual work of running jobs.
6
+ # Since both the worker's thread and whatever thread is managing the
7
+ # worker are capable of affecting the state of the worker's thread, we
8
+ # need to synchronize access to it.
27
9
 
28
- # Default worker priorities. Rule of thumb: number of lowest-priority
29
- # workers should equal number of processors available to us.
30
- PRIORITIES = [5, 5, 5, 5, 4, 3, 2, 1].freeze
31
-
32
- # Which errors should signal a worker that it should hold off before trying
33
- # to grab another job, in order to avoid spamming the logs.
34
- DELAYABLE_ERRORS = %w(
35
- Sequel::DatabaseConnectionError
36
- Sequel::DatabaseDisconnectError
37
- )
38
-
39
- # How long the wrangler thread should wait between pings of the database.
40
- # Future directions: when we have multiple dynos, add rand() to this value
41
- # in the wrangler loop below, so that the dynos' checks will be spaced out.
42
- SLEEP_PERIOD = 5
43
-
44
- # How long to sleep, in repeated increments, for something to happen.
45
- WAIT_PERIOD = 0.0001 # 0.1 ms
46
-
47
- # How long a worker should wait before trying to get another job, in the
48
- # event of a database connection problem.
49
- ERROR_PERIOD = 5
10
+ include MonitorMixin
50
11
 
51
- attr_reader :thread, :priority
12
+ attr_reader :thread
52
13
 
53
- def initialize(priority)
54
- super() # For MonitorMixin
14
+ def initialize
15
+ super # For MonitorMixin.
55
16
 
56
- # These threads have a bad habit of never even having their directive and
57
- # state set if we do it inside their threads. So instead, force the issue
58
- # by doing it outside and holding them up via a queue until their initial
59
- # state is set.
17
+ # We have to make sure the thread doesn't actually start the work loop
18
+ # until it has a state and directive already set up, so use a queue to
19
+ # temporarily block it.
60
20
  q = Queue.new
61
21
 
62
- @priority = priority
63
22
  @thread = Thread.new do
64
23
  q.pop
65
- job = nil
66
24
 
67
25
  loop do
68
- sleep! unless work_job
26
+ job = Job.work
27
+
28
+ # Grab the lock and figure out what we should do next.
29
+ synchronize do
30
+ if @thread[:directive] == :stop
31
+ @thread[:state] = :stopping
32
+ elsif not job
33
+ # No work, go to sleep.
34
+ @thread[:state] = :sleeping
35
+ end
36
+ end
69
37
 
70
- if @thread[:directive] == :sleep
71
- @thread[:state] = :sleeping
38
+ if @thread[:state] == :sleeping
72
39
  sleep
40
+
41
+ # Now that we're woken up, grab the lock figure out if we're stopping.
42
+ synchronize do
43
+ @thread[:state] = :stopping if @thread[:directive] == :stop
44
+ end
73
45
  end
46
+
47
+ break if @thread[:state] == :stopping
74
48
  end
75
49
  end
76
50
 
77
- # All workers are working when first instantiated.
78
- synchronize { @thread[:directive], @thread[:state] = :work, :working }
79
-
80
- # Now the worker can start.
81
- q.push nil
51
+ synchronize do
52
+ @thread[:directive] = :work
53
+ @thread[:state] = :working
54
+ end
82
55
 
83
- # Default thread priority is 0 - make worker threads a bit less important
84
- # than threads that are handling requests.
85
- @thread.priority = -1
56
+ q.push :go!
86
57
  end
87
58
 
88
- # If the worker is asleep, wakes it up and returns truthy. If it's already
89
- # awake, does nothing and returns falsy.
90
- def wake!
59
+ def sleeping?
91
60
  synchronize do
92
- if sleeping?
93
- # There's a very brief period of time where the worker may be marked
94
- # as sleeping but the thread hasn't actually gone to sleep yet.
95
- wait until @thread.stop?
96
- @thread[:directive] = :work
97
-
98
- # Have to set state here so that another poke immediately after this
99
- # one doesn't see the current state as sleeping.
100
- @thread[:state] = :working
101
-
102
- # Now it's safe to wake up the worker.
103
- @thread.wakeup
61
+ if @thread[:state] == :sleeping
62
+ # There's a very small period of time between when the Worker marks
63
+ # itself as sleeping and when it actually goes to sleep. Only report
64
+ # true when we're certain the thread is sleeping.
65
+ sleep 0.0001 until @thread.status == 'sleep'
66
+ true
104
67
  end
105
68
  end
106
69
  end
107
70
 
108
- def sleep!
109
- synchronize { @thread[:directive] = :sleep }
71
+ def working?
72
+ synchronize do
73
+ @thread[:state] == :working
74
+ end
110
75
  end
111
76
 
112
- def awake?
77
+ def wake!
113
78
  synchronize do
114
- %w(sleeping working).include?(@thread[:state].to_s) &&
115
- %w(sleep work).include?(@thread[:directive].to_s) &&
116
- %w(sleep run).include?(@thread.status)
79
+ if sleeping?
80
+ # Have to set the state here so that another thread checking
81
+ # immediately after this won't see the worker as asleep.
82
+ @thread[:state] = :working
83
+ @thread.wakeup
84
+ true
85
+ end
117
86
  end
118
87
  end
119
88
 
120
- def wait_for_sleep
121
- wait until synchronize { sleeping? }
89
+ # This has to be called when trapping a SIGTERM, so it can't lock the monitor.
90
+ def stop!
91
+ @thread[:directive] = :stop
92
+ @thread.wakeup
122
93
  end
123
94
 
124
- private
125
-
126
- def work_job
127
- Job.work(:priority => priority)
128
- rescue => error
129
- self.class.notify_error "Worker error!", error
130
- sleep ERROR_PERIOD if DELAYABLE_ERRORS.include?(error.class.to_s)
131
- return true # There's work available.
95
+ def wait_until_stopped
96
+ @thread.join
132
97
  end
133
98
 
134
- def sleeping?
135
- @thread[:state] == :sleeping
136
- end
99
+ private
137
100
 
138
- def wait
139
- sleep WAIT_PERIOD
140
- end
101
+ # Defaults for the Worker pool.
102
+ @worker_count = 0
103
+ @sleep_period = 5
141
104
 
142
- # The Worker class is responsible for managing the worker instances.
143
105
  class << self
144
- def state=(state)
145
- synchronize do
146
- Que.logger.info "Setting Worker to #{state}..." if Que.logger
147
- case state
148
- when :async
149
- # If this is the first time starting up Worker, start up all workers
150
- # immediately, for the case of a restart during heavy app usage.
151
- workers
152
- # Make sure the wrangler thread is running, it'll do the rest.
153
- @wrangler ||= Thread.new { loop { wrangle } }
154
- when :sync, :off
155
- # Put all the workers to sleep.
156
- workers.each(&:sleep!).each(&:wait_for_sleep)
157
- else
158
- raise "Bad Worker state! #{state.inspect}"
159
- end
160
-
161
- Que.logger.info "Set Worker to #{state}" if Que.logger
162
- @state = state
106
+ attr_reader :mode, :sleep_period, :worker_count
107
+
108
+ def mode=(mode)
109
+ case mode
110
+ when :async
111
+ wrangler # Make sure the wrangler thread is initialized.
112
+ self.worker_count = 4
113
+ else
114
+ self.worker_count = 0
163
115
  end
164
- end
165
116
 
166
- def state
167
- synchronize { @state ||= :off }
117
+ @mode = mode
118
+ Que.log :info, "Set mode to #{mode.inspect}"
168
119
  end
169
120
 
170
- def async?
171
- state == :async
121
+ def workers
122
+ @workers ||= []
172
123
  end
173
124
 
174
- # All workers are up and processing jobs?
175
- def up?(*states)
176
- synchronize { async? && workers.map(&:priority) == PRIORITIES && workers.all?(&:awake?) }
125
+ def worker_count=(count)
126
+ if count > workers.count
127
+ (count - workers.count).times { workers << new }
128
+ elsif count < workers.count
129
+ workers.pop(workers.count - count).each(&:stop!).each(&:wait_until_stopped)
130
+ end
177
131
  end
178
132
 
179
- def workers
180
- @workers || synchronize { @workers ||= PRIORITIES.map { |i| new(i) } }
133
+ def sleep_period=(period)
134
+ @sleep_period = period
135
+ wrangler.wakeup if period
181
136
  end
182
137
 
183
- # Wake up just one worker to work a job, if running async.
184
138
  def wake!
185
- synchronize { async? && workers.find(&:wake!) }
139
+ workers.find &:wake!
186
140
  end
187
141
 
188
- def notify_error(message, error)
189
- log_error message, error
190
- #ExceptionNotifier.notify_exception(error)
191
- rescue => error
192
- log_error "Error notification error!", error
142
+ def wake_all!
143
+ workers.each &:wake!
193
144
  end
194
145
 
195
146
  private
196
147
 
197
- # The wrangler runs this method continuously.
198
- def wrangle
199
- sleep SLEEP_PERIOD
200
- wake!
201
- rescue => error
202
- notify_error "Wrangler Error!", error
203
- end
204
-
205
- def log_error(message, error)
206
- if Que.logger
207
- Que.logger.error <<-ERROR
208
- #{message}
209
- #{error.message}
210
- #{error.backtrace.join("\n")}
211
- ERROR
212
- end
148
+ def wrangler
149
+ @wrangler ||= Thread.new { loop { sleep(*sleep_period); wake! } }
213
150
  end
214
151
  end
215
152
  end