faktory_worker_ruby 0.7.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.travis.yml +11 -0
- data/Changes.md +27 -0
- data/Gemfile +2 -0
- data/Gemfile.lock +34 -8
- data/README.md +10 -5
- data/faktory_worker_ruby.gemspec +4 -3
- data/lib/active_job/queue_adapters/faktory_adapter.rb +61 -0
- data/lib/faktory.rb +3 -1
- data/lib/faktory/batch.rb +178 -0
- data/lib/faktory/cli.rb +9 -0
- data/lib/faktory/client.rb +158 -35
- data/lib/faktory/connection.rb +1 -1
- data/lib/faktory/io.rb +51 -0
- data/lib/faktory/job.rb +41 -29
- data/lib/faktory/launcher.rb +20 -5
- data/lib/faktory/logging.rb +2 -2
- data/lib/faktory/manager.rb +3 -0
- data/lib/faktory/middleware/batch.rb +38 -0
- data/lib/faktory/middleware/i18n.rb +2 -4
- data/lib/faktory/mutate.rb +85 -0
- data/lib/faktory/processor.rb +6 -4
- data/lib/faktory/rails.rb +31 -0
- data/lib/faktory/testing.rb +3 -7
- data/lib/faktory/tracking.rb +41 -0
- data/lib/faktory/version.rb +1 -1
- metadata +34 -14
data/lib/faktory/cli.rb
CHANGED
@@ -8,6 +8,11 @@ require 'optparse'
|
|
8
8
|
require 'erb'
|
9
9
|
require 'fileutils'
|
10
10
|
|
11
|
+
module Faktory
|
12
|
+
class CLI
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
11
16
|
require 'faktory'
|
12
17
|
require 'faktory/util'
|
13
18
|
|
@@ -235,6 +240,10 @@ module Faktory
|
|
235
240
|
opts[:tag] = arg
|
236
241
|
end
|
237
242
|
|
243
|
+
o.on '-l', '--label LABEL', "Process label to use in Faktory UI" do |arg|
|
244
|
+
(opts[:labels] ||= []) << arg
|
245
|
+
end
|
246
|
+
|
238
247
|
o.on "-q", "--queue QUEUE[,WEIGHT]", "Queues to process with optional weights" do |arg|
|
239
248
|
queue, weight = arg.split(",")
|
240
249
|
parse_queue opts, queue, weight
|
data/lib/faktory/client.rb
CHANGED
@@ -1,15 +1,29 @@
|
|
1
1
|
require 'socket'
|
2
2
|
require 'json'
|
3
3
|
require 'uri'
|
4
|
+
require 'digest'
|
4
5
|
require 'securerandom'
|
6
|
+
require 'timeout'
|
7
|
+
require 'faktory/io'
|
5
8
|
|
6
9
|
module Faktory
|
7
|
-
class
|
8
|
-
class
|
9
|
-
|
10
|
+
class BaseError < StandardError; end
|
11
|
+
class CommandError < BaseError; end
|
12
|
+
class ParseError < BaseError; end
|
13
|
+
|
14
|
+
# Faktory::Client provides a low-level connection to a Faktory server
|
15
|
+
# and APIs which map to Faktory commands.
|
16
|
+
#
|
17
|
+
# Most APIs will return `true` if the operation succeeded or raise a
|
18
|
+
# Faktory::BaseError if there was an unexpected error.
|
10
19
|
class Client
|
20
|
+
# provides gets() and read() that respect a read timeout
|
21
|
+
include Faktory::ReadTimeout
|
22
|
+
|
11
23
|
@@random_process_wid = ""
|
12
24
|
|
25
|
+
DEFAULT_TIMEOUT = 5.0
|
26
|
+
|
13
27
|
HASHER = proc do |iter, pwd, salt|
|
14
28
|
sha = Digest::SHA256.new
|
15
29
|
hashing = pwd + salt
|
@@ -35,10 +49,13 @@ module Faktory
|
|
35
49
|
# MY_FAKTORY_URL=tcp://:somepass@my-server.example.com:7419
|
36
50
|
#
|
37
51
|
# Note above, the URL can contain the password for secure installations.
|
38
|
-
def initialize(url: uri_from_env || 'tcp://localhost:7419', debug: false)
|
52
|
+
def initialize(url: uri_from_env || 'tcp://localhost:7419', debug: false, timeout: DEFAULT_TIMEOUT)
|
53
|
+
super
|
39
54
|
@debug = debug
|
40
55
|
@location = URI(url)
|
41
|
-
|
56
|
+
@timeout = timeout
|
57
|
+
|
58
|
+
open(@timeout)
|
42
59
|
end
|
43
60
|
|
44
61
|
def close
|
@@ -52,23 +69,95 @@ module Faktory
|
|
52
69
|
def flush
|
53
70
|
transaction do
|
54
71
|
command "FLUSH"
|
55
|
-
ok
|
72
|
+
ok
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
def create_batch(batch, &block)
|
77
|
+
bid = transaction do
|
78
|
+
command "BATCH NEW", Faktory.dump_json(batch.to_h)
|
79
|
+
result!
|
80
|
+
end
|
81
|
+
batch.instance_variable_set(:@bid, bid)
|
82
|
+
|
83
|
+
old = Thread.current["faktory_batch"]
|
84
|
+
Thread.current["faktory_batch"] = batch
|
85
|
+
begin
|
86
|
+
# any jobs pushed in this block will implicitly have
|
87
|
+
# their `bid` attribute set so they are associated
|
88
|
+
# with the current batch.
|
89
|
+
yield batch
|
90
|
+
ensure
|
91
|
+
Thread.current[:faktory_batch] = old
|
92
|
+
end
|
93
|
+
transaction do
|
94
|
+
command "BATCH COMMIT", bid
|
95
|
+
ok
|
56
96
|
end
|
97
|
+
bid
|
57
98
|
end
|
58
99
|
|
100
|
+
def batch_status(bid)
|
101
|
+
transaction do
|
102
|
+
command "BATCH STATUS", bid
|
103
|
+
Faktory.load_json result!
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
def reopen_batch(b)
|
108
|
+
transaction do
|
109
|
+
command "BATCH OPEN", b.bid
|
110
|
+
ok
|
111
|
+
end
|
112
|
+
old = Thread.current[:faktory_batch]
|
113
|
+
Thread.current[:faktory_batch] = b
|
114
|
+
begin
|
115
|
+
# any jobs pushed in this block will implicitly have
|
116
|
+
# their `bid` attribute set so they are associated
|
117
|
+
# with the current batch.
|
118
|
+
yield b
|
119
|
+
ensure
|
120
|
+
Thread.current[:faktory_batch] = old
|
121
|
+
end
|
122
|
+
transaction do
|
123
|
+
command "BATCH COMMIT", b.bid
|
124
|
+
ok
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
def get_track(jid)
|
129
|
+
transaction do
|
130
|
+
command "TRACK GET", jid
|
131
|
+
hashstr = result!
|
132
|
+
JSON.parse(hashstr)
|
133
|
+
end
|
134
|
+
end
|
135
|
+
|
136
|
+
# hash must include a 'jid' element
|
137
|
+
def set_track(hash)
|
138
|
+
transaction do
|
139
|
+
command("TRACK SET", Faktory.dump_json(hash))
|
140
|
+
ok
|
141
|
+
end
|
142
|
+
end
|
143
|
+
|
144
|
+
# Push a hash corresponding to a job payload to Faktory.
|
145
|
+
# Hash must contain "jid", "jobtype" and "args" elements at minimum.
|
146
|
+
# Returned value will either be the JID String if successful OR
|
147
|
+
# a symbol corresponding to an error.
|
59
148
|
def push(job)
|
60
149
|
transaction do
|
61
|
-
command "PUSH",
|
62
|
-
ok
|
63
|
-
job["jid"]
|
150
|
+
command "PUSH", Faktory.dump_json(job)
|
151
|
+
ok(job["jid"])
|
64
152
|
end
|
65
153
|
end
|
66
154
|
|
155
|
+
# Returns either a job hash or falsy.
|
67
156
|
def fetch(*queues)
|
68
157
|
job = nil
|
69
158
|
transaction do
|
70
159
|
command("FETCH", *queues)
|
71
|
-
job = result
|
160
|
+
job = result!
|
72
161
|
end
|
73
162
|
JSON.parse(job) if job
|
74
163
|
end
|
@@ -76,34 +165,42 @@ module Faktory
|
|
76
165
|
def ack(jid)
|
77
166
|
transaction do
|
78
167
|
command("ACK", %Q[{"jid":"#{jid}"}])
|
79
|
-
ok
|
168
|
+
ok
|
80
169
|
end
|
81
170
|
end
|
82
171
|
|
83
172
|
def fail(jid, ex)
|
84
173
|
transaction do
|
85
|
-
command("FAIL",
|
174
|
+
command("FAIL", Faktory.dump_json({ message: ex.message[0...1000],
|
86
175
|
errtype: ex.class.name,
|
87
176
|
jid: jid,
|
88
177
|
backtrace: ex.backtrace}))
|
89
|
-
ok
|
178
|
+
ok
|
90
179
|
end
|
91
180
|
end
|
92
181
|
|
93
182
|
# Sends a heartbeat to the server, in order to prove this
|
94
183
|
# worker process is still alive.
|
95
184
|
#
|
185
|
+
# You can pass in the current_state of the process, for example during shutdown
|
186
|
+
# quiet and/or terminate can be supplied.
|
187
|
+
#
|
96
188
|
# Return a string signal to process, legal values are "quiet" or "terminate".
|
97
189
|
# The quiet signal is informative: the server won't allow this process to FETCH
|
98
190
|
# any more jobs anyways.
|
99
|
-
def beat
|
191
|
+
def beat(current_state = nil)
|
100
192
|
transaction do
|
101
|
-
|
102
|
-
|
193
|
+
if current_state.nil?
|
194
|
+
command("BEAT", %Q[{"wid":"#{@@random_process_wid}"}])
|
195
|
+
else
|
196
|
+
command("BEAT", %Q[{"wid":"#{@@random_process_wid}", "current_state":"#{current_state}"}])
|
197
|
+
end
|
198
|
+
|
199
|
+
str = result!
|
103
200
|
if str == "OK"
|
104
201
|
str
|
105
202
|
else
|
106
|
-
hash =
|
203
|
+
hash = Faktory.load_json(str)
|
107
204
|
hash["state"]
|
108
205
|
end
|
109
206
|
end
|
@@ -112,8 +209,8 @@ module Faktory
|
|
112
209
|
def info
|
113
210
|
transaction do
|
114
211
|
command("INFO")
|
115
|
-
str = result
|
116
|
-
|
212
|
+
str = result!
|
213
|
+
Faktory.load_json(str) if str
|
117
214
|
end
|
118
215
|
end
|
119
216
|
|
@@ -128,12 +225,14 @@ module Faktory
|
|
128
225
|
@location.scheme =~ /tls/
|
129
226
|
end
|
130
227
|
|
131
|
-
def open
|
228
|
+
def open(timeout = DEFAULT_TIMEOUT)
|
132
229
|
if tls?
|
133
230
|
sock = TCPSocket.new(@location.hostname, @location.port)
|
231
|
+
sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, true)
|
232
|
+
|
134
233
|
ctx = OpenSSL::SSL::SSLContext.new
|
135
234
|
ctx.set_params(verify_mode: OpenSSL::SSL::VERIFY_PEER)
|
136
|
-
ctx.
|
235
|
+
ctx.min_version = OpenSSL::SSL::TLS1_2_VERSION
|
137
236
|
|
138
237
|
@sock = OpenSSL::SSL::SSLSocket.new(sock, ctx).tap do |socket|
|
139
238
|
socket.sync_close = true
|
@@ -148,7 +247,7 @@ module Faktory
|
|
148
247
|
"wid": @@random_process_wid,
|
149
248
|
"hostname": Socket.gethostname,
|
150
249
|
"pid": $$,
|
151
|
-
"labels": ["ruby-#{RUBY_VERSION}"],
|
250
|
+
"labels": Faktory.options[:labels] || ["ruby-#{RUBY_VERSION}"],
|
152
251
|
"v": 2,
|
153
252
|
}
|
154
253
|
|
@@ -175,11 +274,10 @@ module Faktory
|
|
175
274
|
end
|
176
275
|
end
|
177
276
|
|
178
|
-
command("HELLO",
|
179
|
-
ok
|
277
|
+
command("HELLO", Faktory.dump_json(payload))
|
278
|
+
ok
|
180
279
|
end
|
181
280
|
|
182
|
-
|
183
281
|
def command(*args)
|
184
282
|
cmd = args.join(" ")
|
185
283
|
@sock.puts(cmd)
|
@@ -188,12 +286,22 @@ module Faktory
|
|
188
286
|
|
189
287
|
def transaction
|
190
288
|
retryable = true
|
289
|
+
|
290
|
+
# When using Faktory::Testing, you can get a client which does not actually
|
291
|
+
# have an underlying socket. Now if you disable testing and try to use that
|
292
|
+
# client, it will crash without a socket. This open() handles that case to
|
293
|
+
# transparently open a socket.
|
294
|
+
open(@timeout) if !@sock
|
295
|
+
|
191
296
|
begin
|
192
297
|
yield
|
193
|
-
rescue
|
298
|
+
rescue SystemCallError, SocketError, TimeoutError
|
194
299
|
if retryable
|
195
300
|
retryable = false
|
196
|
-
|
301
|
+
|
302
|
+
@sock.close rescue nil
|
303
|
+
@sock = nil
|
304
|
+
open(@timeout)
|
197
305
|
retry
|
198
306
|
else
|
199
307
|
raise
|
@@ -204,7 +312,7 @@ module Faktory
|
|
204
312
|
# I love pragmatic, simple protocols. Thanks antirez!
|
205
313
|
# https://redis.io/topics/protocol
|
206
314
|
def result
|
207
|
-
line =
|
315
|
+
line = gets
|
208
316
|
debug "< #{line}" if @debug
|
209
317
|
raise Errno::ECONNRESET, "No response" unless line
|
210
318
|
chr = line[0]
|
@@ -213,11 +321,20 @@ module Faktory
|
|
213
321
|
elsif chr == '$'
|
214
322
|
count = line[1..-1].strip.to_i
|
215
323
|
return nil if count == -1
|
216
|
-
data =
|
217
|
-
line =
|
324
|
+
data = read(count) if count > 0
|
325
|
+
line = gets # read extra linefeeds
|
218
326
|
data
|
219
327
|
elsif chr == '-'
|
220
|
-
|
328
|
+
# Server can respond with:
|
329
|
+
#
|
330
|
+
# -ERR Something unexpected
|
331
|
+
# We raise a CommandError
|
332
|
+
#
|
333
|
+
# -NOTUNIQUE Job not unique
|
334
|
+
# We return ["NOTUNIQUE", "Job not unique"]
|
335
|
+
err = line[1..-1].split(" ", 2)
|
336
|
+
raise CommandError, err[1] if err[0] == "ERR"
|
337
|
+
err
|
221
338
|
else
|
222
339
|
# this is bad, indicates we need to reset the socket
|
223
340
|
# and start fresh
|
@@ -225,10 +342,17 @@ module Faktory
|
|
225
342
|
end
|
226
343
|
end
|
227
344
|
|
228
|
-
def ok
|
345
|
+
def ok(retval=true)
|
229
346
|
resp = result
|
230
|
-
|
231
|
-
|
347
|
+
return retval if resp == "OK"
|
348
|
+
return resp[0].to_sym
|
349
|
+
end
|
350
|
+
|
351
|
+
def result!
|
352
|
+
resp = result
|
353
|
+
return nil if resp == nil
|
354
|
+
raise CommandError, resp[0] if !resp.is_a?(String)
|
355
|
+
resp
|
232
356
|
end
|
233
357
|
|
234
358
|
# FAKTORY_PROVIDER=MY_FAKTORY_URL
|
@@ -252,4 +376,3 @@ module Faktory
|
|
252
376
|
|
253
377
|
end
|
254
378
|
end
|
255
|
-
|
data/lib/faktory/connection.rb
CHANGED
data/lib/faktory/io.rb
ADDED
@@ -0,0 +1,51 @@
|
|
1
|
+
require "io/wait"
|
2
|
+
|
3
|
+
# this is the necessary magic to get a line-oriented protocol to
|
4
|
+
# respect a read timeout. unfortunately Ruby sockets do not provide any
|
5
|
+
# timeout support directly, delegating that to the IO reactor.
|
6
|
+
module Faktory
|
7
|
+
class TimeoutError < Timeout::Error; end
|
8
|
+
|
9
|
+
module ReadTimeout
|
10
|
+
CRLF = "\r\n"
|
11
|
+
BUFSIZE = 16_384
|
12
|
+
|
13
|
+
# Ruby's TCP sockets do not implement timeouts.
|
14
|
+
# We have to implement them ourselves by using
|
15
|
+
# nonblocking IO and IO.select.
|
16
|
+
def initialize(**opts)
|
17
|
+
@buf = "".dup
|
18
|
+
@timeout = opts[:timeout] || 5
|
19
|
+
end
|
20
|
+
|
21
|
+
def gets
|
22
|
+
while (crlf = @buf.index(CRLF)).nil?
|
23
|
+
@buf << read_timeout(BUFSIZE)
|
24
|
+
end
|
25
|
+
|
26
|
+
@buf.slice!(0, crlf + 2)
|
27
|
+
end
|
28
|
+
|
29
|
+
def read(nbytes)
|
30
|
+
result = @buf.slice!(0, nbytes)
|
31
|
+
result << read_timeout(nbytes - result.bytesize) while result.bytesize < nbytes
|
32
|
+
result
|
33
|
+
end
|
34
|
+
|
35
|
+
private
|
36
|
+
def read_timeout(nbytes)
|
37
|
+
loop do
|
38
|
+
result = @sock.read_nonblock(nbytes, exception: false)
|
39
|
+
if result == :wait_readable
|
40
|
+
raise Faktory::TimeoutError unless @sock.wait_readable(@timeout)
|
41
|
+
elsif result == :wait_writable
|
42
|
+
raise Faktory::TimeoutError unless @sock.wait_writeable(@timeout)
|
43
|
+
elsif result == nil
|
44
|
+
raise Errno::ECONNRESET
|
45
|
+
else
|
46
|
+
return result
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
data/lib/faktory/job.rb
CHANGED
@@ -1,4 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
require 'faktory/tracking'
|
3
|
+
|
2
4
|
module Faktory
|
3
5
|
|
4
6
|
##
|
@@ -20,6 +22,9 @@ module Faktory
|
|
20
22
|
# Note that perform_async is a class method, perform is an instance method.
|
21
23
|
module Job
|
22
24
|
attr_accessor :jid
|
25
|
+
attr_accessor :bid
|
26
|
+
|
27
|
+
include Faktory::Trackable
|
23
28
|
|
24
29
|
def self.included(base)
|
25
30
|
raise ArgumentError, "You cannot include Faktory::Job in an ActiveJob: #{base.name}" if base.ancestors.any? {|c| c.name == 'ActiveJob::Base' }
|
@@ -28,6 +33,16 @@ module Faktory
|
|
28
33
|
base.faktory_class_attribute :faktory_options_hash
|
29
34
|
end
|
30
35
|
|
36
|
+
def self.set(options)
|
37
|
+
Setter.new(options)
|
38
|
+
end
|
39
|
+
|
40
|
+
def batch
|
41
|
+
if bid
|
42
|
+
@batch ||= Faktory::Batch.new(bid)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
31
46
|
def logger
|
32
47
|
Faktory.logger
|
33
48
|
end
|
@@ -42,7 +57,7 @@ module Faktory
|
|
42
57
|
end
|
43
58
|
|
44
59
|
def perform_async(*args)
|
45
|
-
|
60
|
+
client_push(@opts.merge('args'.freeze => args))
|
46
61
|
end
|
47
62
|
|
48
63
|
# +interval+ must be a timestamp, numeric or something that acts
|
@@ -53,12 +68,32 @@ module Faktory
|
|
53
68
|
ts = (int < 1_000_000_000 ? now + int : int)
|
54
69
|
at = Time.at(ts).utc.to_datetime.rfc3339(9)
|
55
70
|
|
56
|
-
@opts.merge
|
71
|
+
item = @opts.merge('args'.freeze => args, 'at'.freeze => at)
|
72
|
+
|
57
73
|
# Optimization to enqueue something now that is scheduled to go out now or in the past
|
58
|
-
|
59
|
-
|
74
|
+
item.delete('at'.freeze) if ts <= now
|
75
|
+
|
76
|
+
client_push(item)
|
60
77
|
end
|
61
78
|
alias_method :perform_at, :perform_in
|
79
|
+
|
80
|
+
def client_push(item) # :nodoc:
|
81
|
+
# stringify
|
82
|
+
item.keys.each do |key|
|
83
|
+
item[key.to_s] = item.delete(key)
|
84
|
+
end
|
85
|
+
item["jid"] ||= SecureRandom.hex(12)
|
86
|
+
item["queue"] ||= "default"
|
87
|
+
|
88
|
+
pool = Thread.current[:faktory_via_pool] || item["pool"] || Faktory.server_pool
|
89
|
+
item.delete("pool")
|
90
|
+
|
91
|
+
Faktory.client_middleware.invoke(item, pool) do
|
92
|
+
pool.with do |c|
|
93
|
+
c.push(item)
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
62
97
|
end
|
63
98
|
|
64
99
|
module ClassMethods
|
@@ -68,19 +103,13 @@ module Faktory
|
|
68
103
|
end
|
69
104
|
|
70
105
|
def perform_async(*args)
|
71
|
-
|
106
|
+
set(get_faktory_options).perform_async(*args)
|
72
107
|
end
|
73
108
|
|
74
109
|
# +interval+ must be a timestamp, numeric or something that acts
|
75
110
|
# numeric (like an activesupport time interval).
|
76
111
|
def perform_in(interval, *args)
|
77
|
-
|
78
|
-
now = Time.now.to_f
|
79
|
-
ts = (int < 1_000_000_000 ? now + int : int)
|
80
|
-
item = { 'jobtype'.freeze => self, 'args'.freeze => args }
|
81
|
-
|
82
|
-
item['at'] = Time.at(ts).utc.to_datetime.rfc3339(9) if ts > now
|
83
|
-
client_push(item)
|
112
|
+
set(get_faktory_options).perform_in(interval, *args)
|
84
113
|
end
|
85
114
|
alias_method :perform_at, :perform_in
|
86
115
|
|
@@ -102,23 +131,6 @@ module Faktory
|
|
102
131
|
self.faktory_options_hash ||= Faktory.default_job_options
|
103
132
|
end
|
104
133
|
|
105
|
-
def client_push(item) # :nodoc:
|
106
|
-
pool = Thread.current[:faktory_via_pool] || get_faktory_options['pool'.freeze] || Faktory.server_pool
|
107
|
-
item = get_faktory_options.merge(item)
|
108
|
-
# stringify
|
109
|
-
item.keys.each do |key|
|
110
|
-
item[key.to_s] = item.delete(key)
|
111
|
-
end
|
112
|
-
item["jid"] ||= SecureRandom.hex(12)
|
113
|
-
item["queue"] ||= "default"
|
114
|
-
|
115
|
-
Faktory.client_middleware.invoke(item, pool) do
|
116
|
-
pool.with do |c|
|
117
|
-
c.push(item)
|
118
|
-
end
|
119
|
-
end
|
120
|
-
end
|
121
|
-
|
122
134
|
def faktory_class_attribute(*attrs)
|
123
135
|
instance_reader = true
|
124
136
|
instance_writer = true
|