faktory_worker_ruby 0.7.0 → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/faktory/cli.rb CHANGED
@@ -8,6 +8,11 @@ require 'optparse'
8
8
  require 'erb'
9
9
  require 'fileutils'
10
10
 
11
+ module Faktory
12
+ class CLI
13
+ end
14
+ end
15
+
11
16
  require 'faktory'
12
17
  require 'faktory/util'
13
18
 
@@ -235,6 +240,10 @@ module Faktory
235
240
  opts[:tag] = arg
236
241
  end
237
242
 
243
+ o.on '-l', '--label LABEL', "Process label to use in Faktory UI" do |arg|
244
+ (opts[:labels] ||= []) << arg
245
+ end
246
+
238
247
  o.on "-q", "--queue QUEUE[,WEIGHT]", "Queues to process with optional weights" do |arg|
239
248
  queue, weight = arg.split(",")
240
249
  parse_queue opts, queue, weight
@@ -1,15 +1,29 @@
1
1
  require 'socket'
2
2
  require 'json'
3
3
  require 'uri'
4
+ require 'digest'
4
5
  require 'securerandom'
6
+ require 'timeout'
7
+ require 'faktory/io'
5
8
 
6
9
  module Faktory
7
- class CommandError < StandardError;end
8
- class ParseError < StandardError;end
9
-
10
+ class BaseError < StandardError; end
11
+ class CommandError < BaseError; end
12
+ class ParseError < BaseError; end
13
+
14
+ # Faktory::Client provides a low-level connection to a Faktory server
15
+ # and APIs which map to Faktory commands.
16
+ #
17
+ # Most APIs will return `true` if the operation succeeded or raise a
18
+ # Faktory::BaseError if there was an unexpected error.
10
19
  class Client
20
+ # provides gets() and read() that respect a read timeout
21
+ include Faktory::ReadTimeout
22
+
11
23
  @@random_process_wid = ""
12
24
 
25
+ DEFAULT_TIMEOUT = 5.0
26
+
13
27
  HASHER = proc do |iter, pwd, salt|
14
28
  sha = Digest::SHA256.new
15
29
  hashing = pwd + salt
@@ -35,10 +49,13 @@ module Faktory
35
49
  # MY_FAKTORY_URL=tcp://:somepass@my-server.example.com:7419
36
50
  #
37
51
  # Note above, the URL can contain the password for secure installations.
38
- def initialize(url: uri_from_env || 'tcp://localhost:7419', debug: false)
52
+ def initialize(url: uri_from_env || 'tcp://localhost:7419', debug: false, timeout: DEFAULT_TIMEOUT)
53
+ super
39
54
  @debug = debug
40
55
  @location = URI(url)
41
- open
56
+ @timeout = timeout
57
+
58
+ open(@timeout)
42
59
  end
43
60
 
44
61
  def close
@@ -52,23 +69,95 @@ module Faktory
52
69
  def flush
53
70
  transaction do
54
71
  command "FLUSH"
55
- ok!
72
+ ok
73
+ end
74
+ end
75
+
76
+ def create_batch(batch, &block)
77
+ bid = transaction do
78
+ command "BATCH NEW", Faktory.dump_json(batch.to_h)
79
+ result!
80
+ end
81
+ batch.instance_variable_set(:@bid, bid)
82
+
83
+ old = Thread.current["faktory_batch"]
84
+ Thread.current["faktory_batch"] = batch
85
+ begin
86
+ # any jobs pushed in this block will implicitly have
87
+ # their `bid` attribute set so they are associated
88
+ # with the current batch.
89
+ yield batch
90
+ ensure
91
+ Thread.current[:faktory_batch] = old
92
+ end
93
+ transaction do
94
+ command "BATCH COMMIT", bid
95
+ ok
56
96
  end
97
+ bid
57
98
  end
58
99
 
100
+ def batch_status(bid)
101
+ transaction do
102
+ command "BATCH STATUS", bid
103
+ Faktory.load_json result!
104
+ end
105
+ end
106
+
107
+ def reopen_batch(b)
108
+ transaction do
109
+ command "BATCH OPEN", b.bid
110
+ ok
111
+ end
112
+ old = Thread.current[:faktory_batch]
113
+ Thread.current[:faktory_batch] = b
114
+ begin
115
+ # any jobs pushed in this block will implicitly have
116
+ # their `bid` attribute set so they are associated
117
+ # with the current batch.
118
+ yield b
119
+ ensure
120
+ Thread.current[:faktory_batch] = old
121
+ end
122
+ transaction do
123
+ command "BATCH COMMIT", b.bid
124
+ ok
125
+ end
126
+ end
127
+
128
+ def get_track(jid)
129
+ transaction do
130
+ command "TRACK GET", jid
131
+ hashstr = result!
132
+ JSON.parse(hashstr)
133
+ end
134
+ end
135
+
136
+ # hash must include a 'jid' element
137
+ def set_track(hash)
138
+ transaction do
139
+ command("TRACK SET", Faktory.dump_json(hash))
140
+ ok
141
+ end
142
+ end
143
+
144
+ # Push a hash corresponding to a job payload to Faktory.
145
+ # Hash must contain "jid", "jobtype" and "args" elements at minimum.
146
+ # Returned value will either be the JID String if successful OR
147
+ # a symbol corresponding to an error.
59
148
  def push(job)
60
149
  transaction do
61
- command "PUSH", JSON.generate(job)
62
- ok!
63
- job["jid"]
150
+ command "PUSH", Faktory.dump_json(job)
151
+ ok(job["jid"])
64
152
  end
65
153
  end
66
154
 
155
+ # Returns either a job hash or falsy.
67
156
  def fetch(*queues)
68
157
  job = nil
69
158
  transaction do
70
159
  command("FETCH", *queues)
71
- job = result
160
+ job = result!
72
161
  end
73
162
  JSON.parse(job) if job
74
163
  end
@@ -76,34 +165,42 @@ module Faktory
76
165
  def ack(jid)
77
166
  transaction do
78
167
  command("ACK", %Q[{"jid":"#{jid}"}])
79
- ok!
168
+ ok
80
169
  end
81
170
  end
82
171
 
83
172
  def fail(jid, ex)
84
173
  transaction do
85
- command("FAIL", JSON.dump({ message: ex.message[0...1000],
174
+ command("FAIL", Faktory.dump_json({ message: ex.message[0...1000],
86
175
  errtype: ex.class.name,
87
176
  jid: jid,
88
177
  backtrace: ex.backtrace}))
89
- ok!
178
+ ok
90
179
  end
91
180
  end
92
181
 
93
182
  # Sends a heartbeat to the server, in order to prove this
94
183
  # worker process is still alive.
95
184
  #
185
+ # You can pass in the current_state of the process, for example during shutdown
186
+ # quiet and/or terminate can be supplied.
187
+ #
96
188
  # Return a string signal to process, legal values are "quiet" or "terminate".
97
189
  # The quiet signal is informative: the server won't allow this process to FETCH
98
190
  # any more jobs anyways.
99
- def beat
191
+ def beat(current_state = nil)
100
192
  transaction do
101
- command("BEAT", %Q[{"wid":"#{@@random_process_wid}"}])
102
- str = result
193
+ if current_state.nil?
194
+ command("BEAT", %Q[{"wid":"#{@@random_process_wid}"}])
195
+ else
196
+ command("BEAT", %Q[{"wid":"#{@@random_process_wid}", "current_state":"#{current_state}"}])
197
+ end
198
+
199
+ str = result!
103
200
  if str == "OK"
104
201
  str
105
202
  else
106
- hash = JSON.parse(str)
203
+ hash = Faktory.load_json(str)
107
204
  hash["state"]
108
205
  end
109
206
  end
@@ -112,8 +209,8 @@ module Faktory
112
209
  def info
113
210
  transaction do
114
211
  command("INFO")
115
- str = result
116
- JSON.parse(str) if str
212
+ str = result!
213
+ Faktory.load_json(str) if str
117
214
  end
118
215
  end
119
216
 
@@ -128,12 +225,14 @@ module Faktory
128
225
  @location.scheme =~ /tls/
129
226
  end
130
227
 
131
- def open
228
+ def open(timeout = DEFAULT_TIMEOUT)
132
229
  if tls?
133
230
  sock = TCPSocket.new(@location.hostname, @location.port)
231
+ sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, true)
232
+
134
233
  ctx = OpenSSL::SSL::SSLContext.new
135
234
  ctx.set_params(verify_mode: OpenSSL::SSL::VERIFY_PEER)
136
- ctx.ssl_version = :TLSv1_2
235
+ ctx.min_version = OpenSSL::SSL::TLS1_2_VERSION
137
236
 
138
237
  @sock = OpenSSL::SSL::SSLSocket.new(sock, ctx).tap do |socket|
139
238
  socket.sync_close = true
@@ -148,7 +247,7 @@ module Faktory
148
247
  "wid": @@random_process_wid,
149
248
  "hostname": Socket.gethostname,
150
249
  "pid": $$,
151
- "labels": ["ruby-#{RUBY_VERSION}"],
250
+ "labels": Faktory.options[:labels] || ["ruby-#{RUBY_VERSION}"],
152
251
  "v": 2,
153
252
  }
154
253
 
@@ -175,11 +274,10 @@ module Faktory
175
274
  end
176
275
  end
177
276
 
178
- command("HELLO", JSON.dump(payload))
179
- ok!
277
+ command("HELLO", Faktory.dump_json(payload))
278
+ ok
180
279
  end
181
280
 
182
-
183
281
  def command(*args)
184
282
  cmd = args.join(" ")
185
283
  @sock.puts(cmd)
@@ -188,12 +286,22 @@ module Faktory
188
286
 
189
287
  def transaction
190
288
  retryable = true
289
+
290
+ # When using Faktory::Testing, you can get a client which does not actually
291
+ # have an underlying socket. Now if you disable testing and try to use that
292
+ # client, it will crash without a socket. This open() handles that case to
293
+ # transparently open a socket.
294
+ open(@timeout) if !@sock
295
+
191
296
  begin
192
297
  yield
193
- rescue Errno::EPIPE, Errno::ECONNRESET
298
+ rescue SystemCallError, SocketError, TimeoutError
194
299
  if retryable
195
300
  retryable = false
196
- open
301
+
302
+ @sock.close rescue nil
303
+ @sock = nil
304
+ open(@timeout)
197
305
  retry
198
306
  else
199
307
  raise
@@ -204,7 +312,7 @@ module Faktory
204
312
  # I love pragmatic, simple protocols. Thanks antirez!
205
313
  # https://redis.io/topics/protocol
206
314
  def result
207
- line = @sock.gets
315
+ line = gets
208
316
  debug "< #{line}" if @debug
209
317
  raise Errno::ECONNRESET, "No response" unless line
210
318
  chr = line[0]
@@ -213,11 +321,20 @@ module Faktory
213
321
  elsif chr == '$'
214
322
  count = line[1..-1].strip.to_i
215
323
  return nil if count == -1
216
- data = @sock.read(count) if count > 0
217
- line = @sock.gets # read extra linefeeds
324
+ data = read(count) if count > 0
325
+ line = gets # read extra linefeeds
218
326
  data
219
327
  elsif chr == '-'
220
- raise CommandError, line[1..-1]
328
+ # Server can respond with:
329
+ #
330
+ # -ERR Something unexpected
331
+ # We raise a CommandError
332
+ #
333
+ # -NOTUNIQUE Job not unique
334
+ # We return ["NOTUNIQUE", "Job not unique"]
335
+ err = line[1..-1].split(" ", 2)
336
+ raise CommandError, err[1] if err[0] == "ERR"
337
+ err
221
338
  else
222
339
  # this is bad, indicates we need to reset the socket
223
340
  # and start fresh
@@ -225,10 +342,17 @@ module Faktory
225
342
  end
226
343
  end
227
344
 
228
- def ok!
345
+ def ok(retval=true)
229
346
  resp = result
230
- raise CommandError, resp if resp != "OK"
231
- true
347
+ return retval if resp == "OK"
348
+ return resp[0].to_sym
349
+ end
350
+
351
+ def result!
352
+ resp = result
353
+ return nil if resp == nil
354
+ raise CommandError, resp[0] if !resp.is_a?(String)
355
+ resp
232
356
  end
233
357
 
234
358
  # FAKTORY_PROVIDER=MY_FAKTORY_URL
@@ -252,4 +376,3 @@ module Faktory
252
376
 
253
377
  end
254
378
  end
255
-
@@ -7,7 +7,7 @@ module Faktory
7
7
  def create(options={})
8
8
  size = Faktory.worker? ? (Faktory.options[:concurrency] + 2) : 5
9
9
  ConnectionPool.new(:timeout => options[:pool_timeout] || 1, :size => size) do
10
- Faktory::Client.new
10
+ Faktory::Client.new(**options)
11
11
  end
12
12
  end
13
13
  end
data/lib/faktory/io.rb ADDED
@@ -0,0 +1,51 @@
1
+ require "io/wait"
2
+
3
+ # this is the necessary magic to get a line-oriented protocol to
4
+ # respect a read timeout. unfortunately Ruby sockets do not provide any
5
+ # timeout support directly, delegating that to the IO reactor.
6
+ module Faktory
7
+ class TimeoutError < Timeout::Error; end
8
+
9
+ module ReadTimeout
10
+ CRLF = "\r\n"
11
+ BUFSIZE = 16_384
12
+
13
+ # Ruby's TCP sockets do not implement timeouts.
14
+ # We have to implement them ourselves by using
15
+ # nonblocking IO and IO.select.
16
+ def initialize(**opts)
17
+ @buf = "".dup
18
+ @timeout = opts[:timeout] || 5
19
+ end
20
+
21
+ def gets
22
+ while (crlf = @buf.index(CRLF)).nil?
23
+ @buf << read_timeout(BUFSIZE)
24
+ end
25
+
26
+ @buf.slice!(0, crlf + 2)
27
+ end
28
+
29
+ def read(nbytes)
30
+ result = @buf.slice!(0, nbytes)
31
+ result << read_timeout(nbytes - result.bytesize) while result.bytesize < nbytes
32
+ result
33
+ end
34
+
35
+ private
36
+ def read_timeout(nbytes)
37
+ loop do
38
+ result = @sock.read_nonblock(nbytes, exception: false)
39
+ if result == :wait_readable
40
+ raise Faktory::TimeoutError unless @sock.wait_readable(@timeout)
41
+ elsif result == :wait_writable
42
+ raise Faktory::TimeoutError unless @sock.wait_writeable(@timeout)
43
+ elsif result == nil
44
+ raise Errno::ECONNRESET
45
+ else
46
+ return result
47
+ end
48
+ end
49
+ end
50
+ end
51
+ end
data/lib/faktory/job.rb CHANGED
@@ -1,4 +1,6 @@
1
1
  # frozen_string_literal: true
2
+ require 'faktory/tracking'
3
+
2
4
  module Faktory
3
5
 
4
6
  ##
@@ -20,6 +22,9 @@ module Faktory
20
22
  # Note that perform_async is a class method, perform is an instance method.
21
23
  module Job
22
24
  attr_accessor :jid
25
+ attr_accessor :bid
26
+
27
+ include Faktory::Trackable
23
28
 
24
29
  def self.included(base)
25
30
  raise ArgumentError, "You cannot include Faktory::Job in an ActiveJob: #{base.name}" if base.ancestors.any? {|c| c.name == 'ActiveJob::Base' }
@@ -28,6 +33,16 @@ module Faktory
28
33
  base.faktory_class_attribute :faktory_options_hash
29
34
  end
30
35
 
36
+ def self.set(options)
37
+ Setter.new(options)
38
+ end
39
+
40
+ def batch
41
+ if bid
42
+ @batch ||= Faktory::Batch.new(bid)
43
+ end
44
+ end
45
+
31
46
  def logger
32
47
  Faktory.logger
33
48
  end
@@ -42,7 +57,7 @@ module Faktory
42
57
  end
43
58
 
44
59
  def perform_async(*args)
45
- @opts['jobtype'.freeze].client_push(@opts.merge!('args'.freeze => args))
60
+ client_push(@opts.merge('args'.freeze => args))
46
61
  end
47
62
 
48
63
  # +interval+ must be a timestamp, numeric or something that acts
@@ -53,12 +68,32 @@ module Faktory
53
68
  ts = (int < 1_000_000_000 ? now + int : int)
54
69
  at = Time.at(ts).utc.to_datetime.rfc3339(9)
55
70
 
56
- @opts.merge! 'args'.freeze => args, 'at'.freeze => at
71
+ item = @opts.merge('args'.freeze => args, 'at'.freeze => at)
72
+
57
73
  # Optimization to enqueue something now that is scheduled to go out now or in the past
58
- @opts.delete('at'.freeze) if ts <= now
59
- @opts['jobtype'.freeze].client_push(@opts)
74
+ item.delete('at'.freeze) if ts <= now
75
+
76
+ client_push(item)
60
77
  end
61
78
  alias_method :perform_at, :perform_in
79
+
80
+ def client_push(item) # :nodoc:
81
+ # stringify
82
+ item.keys.each do |key|
83
+ item[key.to_s] = item.delete(key)
84
+ end
85
+ item["jid"] ||= SecureRandom.hex(12)
86
+ item["queue"] ||= "default"
87
+
88
+ pool = Thread.current[:faktory_via_pool] || item["pool"] || Faktory.server_pool
89
+ item.delete("pool")
90
+
91
+ Faktory.client_middleware.invoke(item, pool) do
92
+ pool.with do |c|
93
+ c.push(item)
94
+ end
95
+ end
96
+ end
62
97
  end
63
98
 
64
99
  module ClassMethods
@@ -68,19 +103,13 @@ module Faktory
68
103
  end
69
104
 
70
105
  def perform_async(*args)
71
- client_push('jobtype'.freeze => self, 'args'.freeze => args)
106
+ set(get_faktory_options).perform_async(*args)
72
107
  end
73
108
 
74
109
  # +interval+ must be a timestamp, numeric or something that acts
75
110
  # numeric (like an activesupport time interval).
76
111
  def perform_in(interval, *args)
77
- int = interval.to_f
78
- now = Time.now.to_f
79
- ts = (int < 1_000_000_000 ? now + int : int)
80
- item = { 'jobtype'.freeze => self, 'args'.freeze => args }
81
-
82
- item['at'] = Time.at(ts).utc.to_datetime.rfc3339(9) if ts > now
83
- client_push(item)
112
+ set(get_faktory_options).perform_in(interval, *args)
84
113
  end
85
114
  alias_method :perform_at, :perform_in
86
115
 
@@ -102,23 +131,6 @@ module Faktory
102
131
  self.faktory_options_hash ||= Faktory.default_job_options
103
132
  end
104
133
 
105
- def client_push(item) # :nodoc:
106
- pool = Thread.current[:faktory_via_pool] || get_faktory_options['pool'.freeze] || Faktory.server_pool
107
- item = get_faktory_options.merge(item)
108
- # stringify
109
- item.keys.each do |key|
110
- item[key.to_s] = item.delete(key)
111
- end
112
- item["jid"] ||= SecureRandom.hex(12)
113
- item["queue"] ||= "default"
114
-
115
- Faktory.client_middleware.invoke(item, pool) do
116
- pool.with do |c|
117
- c.push(item)
118
- end
119
- end
120
- end
121
-
122
134
  def faktory_class_attribute(*attrs)
123
135
  instance_reader = true
124
136
  instance_writer = true