faktory_worker_ruby 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,296 @@
1
+ # encoding: utf-8
2
+ # frozen_string_literal: true
3
+ $stdout.sync = true
4
+
5
+ require 'yaml'
6
+ require 'singleton'
7
+ require 'optparse'
8
+ require 'erb'
9
+ require 'fileutils'
10
+
11
+ require 'faktory'
12
+ require 'faktory/util'
13
+
14
+ module Faktory
15
+ class CLI
16
+ include Util
17
+ include Singleton unless $TESTING
18
+
19
+ # Used for CLI testing
20
+ attr_accessor :code
21
+ attr_accessor :launcher
22
+ attr_accessor :environment
23
+
24
+ def initialize
25
+ @code = nil
26
+ end
27
+
28
+ def parse(args=ARGV)
29
+ @code = nil
30
+
31
+ setup_options(args)
32
+ initialize_logger
33
+ validate!
34
+ end
35
+
36
+ def jruby?
37
+ defined?(::JRUBY_VERSION)
38
+ end
39
+
40
+ # Code within this method is not tested because it alters
41
+ # global process state irreversibly. PRs which improve the
42
+ # test coverage of Faktory::CLI are welcomed.
43
+ def run
44
+ boot_system
45
+ print_banner
46
+
47
+ self_read, self_write = IO.pipe
48
+ sigs = %w(INT TERM TTIN TSTP)
49
+
50
+ sigs.each do |sig|
51
+ begin
52
+ trap sig do
53
+ self_write.puts(sig)
54
+ end
55
+ rescue ArgumentError
56
+ puts "Signal #{sig} not supported"
57
+ end
58
+ end
59
+
60
+ logger.info "Running in #{RUBY_DESCRIPTION}"
61
+ logger.info Faktory::LICENSE
62
+
63
+ # cache process identity
64
+ Faktory.options[:identity] = identity
65
+
66
+ # Touch middleware so it isn't lazy loaded by multiple threads, #3043
67
+ Faktory.worker_middleware
68
+
69
+ # Before this point, the process is initializing with just the main thread.
70
+ # Starting here the process will now have multiple threads running.
71
+ fire_event(:startup)
72
+
73
+ logger.debug { "Client Middleware: #{Faktory.client_middleware.map(&:klass).join(', ')}" }
74
+ logger.debug { "Execute Middleware: #{Faktory.worker_middleware.map(&:klass).join(', ')}" }
75
+
76
+ logger.info 'Starting processing, hit Ctrl-C to stop' if $stdout.tty?
77
+
78
+ require 'faktory/launcher'
79
+ @launcher = Faktory::Launcher.new(options)
80
+
81
+ begin
82
+ launcher.run
83
+
84
+ while readable_io = IO.select([self_read])
85
+ signal = readable_io.first[0].gets.strip
86
+ handle_signal(signal)
87
+ end
88
+ rescue Interrupt
89
+ logger.info 'Shutting down'
90
+ launcher.stop
91
+ # Explicitly exit so busy Processor threads can't block
92
+ # process shutdown.
93
+ logger.info "Bye!"
94
+ exit(0)
95
+ end
96
+ end
97
+
98
+ def self.banner
99
+ %q{
100
+ INSERT SWEET FAKTORY BANNER HERE
101
+ }
102
+ end
103
+
104
+ def handle_signal(sig)
105
+ Faktory.logger.debug "Got #{sig} signal"
106
+ case sig
107
+ when 'INT'
108
+ raise Interrupt
109
+ when 'TERM'
110
+ # Heroku sends TERM and then waits 30 seconds for process to exit.
111
+ raise Interrupt
112
+ when 'TSTP'
113
+ Faktory.logger.info "Received TSTP, no longer accepting new work"
114
+ launcher.quiet
115
+ when 'TTIN'
116
+ Thread.list.each do |thread|
117
+ Faktory.logger.warn "Thread TID-#{thread.object_id.to_s(36)} #{thread['faktory_label']}"
118
+ if thread.backtrace
119
+ Faktory.logger.warn thread.backtrace.join("\n")
120
+ else
121
+ Faktory.logger.warn "<no backtrace available>"
122
+ end
123
+ end
124
+ end
125
+ end
126
+
127
+ private
128
+
129
+ def print_banner
130
+ # Print logo and banner for development
131
+ if environment == 'development' && $stdout.tty?
132
+ puts "\e[#{31}m"
133
+ puts Faktory::CLI.banner
134
+ puts "\e[0m"
135
+ end
136
+ end
137
+
138
+ def set_environment(cli_env)
139
+ @environment = cli_env || ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
140
+ end
141
+
142
+ alias_method :die, :exit
143
+ alias_method :☠, :exit
144
+
145
+ def setup_options(args)
146
+ opts = parse_options(args)
147
+ set_environment opts[:environment]
148
+
149
+ cfile = opts[:config_file]
150
+ opts = parse_config(cfile).merge(opts) if cfile
151
+
152
+ opts[:strict] = true if opts[:strict].nil?
153
+ opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if !opts[:concurrency] && ENV["RAILS_MAX_THREADS"]
154
+
155
+ options.merge!(opts)
156
+ end
157
+
158
+ def options
159
+ Faktory.options
160
+ end
161
+
162
+ def boot_system
163
+ ENV['RACK_ENV'] = ENV['RAILS_ENV'] = environment
164
+
165
+ raise ArgumentError, "#{options[:require]} does not exist" unless File.exist?(options[:require])
166
+
167
+ if File.directory?(options[:require])
168
+ require 'rails'
169
+ require 'faktory/rails'
170
+ require File.expand_path("#{options[:require]}/config/environment.rb")
171
+ options[:tag] ||= default_tag
172
+ else
173
+ not_required_message = "#{options[:require]} was not required, you should use an explicit path: " +
174
+ "./#{options[:require]} or /path/to/#{options[:require]}"
175
+
176
+ require(options[:require]) || raise(ArgumentError, not_required_message)
177
+ end
178
+ end
179
+
180
+ def default_tag
181
+ dir = ::Rails.root
182
+ name = File.basename(dir)
183
+ if name.to_i != 0 && prevdir = File.dirname(dir) # Capistrano release directory?
184
+ if File.basename(prevdir) == 'releases'
185
+ return File.basename(File.dirname(prevdir))
186
+ end
187
+ end
188
+ name
189
+ end
190
+
191
+ def validate!
192
+ options[:queues] << 'default' if options[:queues].empty?
193
+
194
+ if !File.exist?(options[:require]) ||
195
+ (File.directory?(options[:require]) && !File.exist?("#{options[:require]}/config/application.rb"))
196
+ logger.info "=================================================================="
197
+ logger.info " Please point Faktory to a Rails 5 application or a Ruby file "
198
+ logger.info " to load your worker classes with -r [DIR|FILE]."
199
+ logger.info "=================================================================="
200
+ logger.info @parser
201
+ die(1)
202
+ end
203
+
204
+ [:concurrency, :timeout].each do |opt|
205
+ raise ArgumentError, "#{opt}: #{options[opt]} is not a valid value" if options.has_key?(opt) && options[opt].to_i <= 0
206
+ end
207
+ end
208
+
209
+ def parse_options(argv)
210
+ opts = {}
211
+
212
+ @parser = OptionParser.new do |o|
213
+ o.on '-c', '--concurrency INT', "processor threads to use" do |arg|
214
+ opts[:concurrency] = Integer(arg)
215
+ end
216
+
217
+ o.on '-e', '--environment ENV', "Application environment" do |arg|
218
+ opts[:environment] = arg
219
+ end
220
+
221
+ o.on '-g', '--tag TAG', "Process tag for procline" do |arg|
222
+ opts[:tag] = arg
223
+ end
224
+
225
+ o.on "-q", "--queue QUEUE[,WEIGHT]", "Queues to process with optional weights" do |arg|
226
+ queue, weight = arg.split(",")
227
+ parse_queue opts, queue, weight
228
+ end
229
+
230
+ o.on '-r', '--require [PATH|DIR]', "Location of Rails application with workers or file to require" do |arg|
231
+ opts[:require] = arg
232
+ end
233
+
234
+ o.on '-t', '--timeout NUM', "Shutdown timeout" do |arg|
235
+ opts[:timeout] = Integer(arg)
236
+ end
237
+
238
+ o.on "-v", "--verbose", "Print more verbose output" do |arg|
239
+ opts[:verbose] = arg
240
+ end
241
+
242
+ o.on '-C', '--config PATH', "path to YAML config file" do |arg|
243
+ opts[:config_file] = arg
244
+ end
245
+
246
+ o.on '-V', '--version', "Print version and exit" do |arg|
247
+ puts "Faktory #{Faktory::VERSION}"
248
+ die(0)
249
+ end
250
+ end
251
+
252
+ @parser.banner = "faktory-worker [options]"
253
+ @parser.on_tail "-h", "--help", "Show help" do
254
+ logger.info @parser
255
+ die 1
256
+ end
257
+ @parser.parse!(argv)
258
+
259
+ %w[config/faktory.yml config/faktory.yml.erb].each do |filename|
260
+ opts[:config_file] ||= filename if File.exist?(filename)
261
+ end
262
+
263
+ opts
264
+ end
265
+
266
+ def initialize_logger
267
+ Faktory::Logging.initialize_logger(options[:logfile]) if options[:logfile]
268
+
269
+ Faktory.logger.level = ::Logger::DEBUG if options[:verbose]
270
+ end
271
+
272
+ def parse_config(cfile)
273
+ opts = {}
274
+ if File.exist?(cfile)
275
+ opts = YAML.load(ERB.new(IO.read(cfile)).result) || opts
276
+ opts = opts.merge(opts.delete(environment) || {})
277
+ parse_queues(opts, opts.delete(:queues) || [])
278
+ else
279
+ # allow a non-existent config file so Faktory
280
+ # can be deployed by cap with just the defaults.
281
+ end
282
+ opts
283
+ end
284
+
285
+ def parse_queues(opts, queues_and_weights)
286
+ queues_and_weights.each { |queue_and_weight| parse_queue(opts, *queue_and_weight) }
287
+ end
288
+
289
+ def parse_queue(opts, q, weight=nil)
290
+ [weight.to_i, 1].max.times do
291
+ (opts[:queues] ||= []) << q
292
+ end
293
+ opts[:strict] = false if weight.to_i > 0
294
+ end
295
+ end
296
+ end
@@ -0,0 +1,227 @@
1
+ require 'socket'
2
+ require 'json'
3
+ require 'uri'
4
+ require 'securerandom'
5
+
6
+ module Faktory
7
+ class CommandError < StandardError;end
8
+ class ParseError < StandardError;end
9
+
10
+ class Client
11
+ @@random_process_wid = SecureRandom.hex(8)
12
+
13
+ attr_accessor :middleware
14
+
15
+ # Best practice is to rely on the localhost default for development
16
+ # and configure the environment variables for non-development environments.
17
+ #
18
+ # FAKTORY_PROVIDER=MY_FAKTORY_URL
19
+ # MY_FAKTORY_URL=tcp://:somepass@my-server.example.com:7419
20
+ #
21
+ # Note above, the URL can contain the password for secure installations.
22
+ def initialize(url: 'tcp://localhost:7419', debug: false)
23
+ @debug = debug
24
+ @location = uri_from_env || URI(url)
25
+ open
26
+ end
27
+
28
+ def close
29
+ return unless @sock
30
+ command "END"
31
+ @sock.close
32
+ @sock = nil
33
+ end
34
+
35
+ # Warning: this clears all job data in Faktory
36
+ def flush
37
+ transaction do
38
+ command "FLUSH"
39
+ ok!
40
+ end
41
+ end
42
+
43
+ def push(job)
44
+ transaction do
45
+ command "PUSH", JSON.generate(job)
46
+ ok!
47
+ job["jid"]
48
+ end
49
+ end
50
+
51
+ def fetch(*queues)
52
+ job = nil
53
+ transaction do
54
+ command("FETCH", *queues)
55
+ job = result
56
+ end
57
+ JSON.parse(job) if job
58
+ end
59
+
60
+ def ack(jid)
61
+ transaction do
62
+ command("ACK", %Q[{"jid":"#{jid}"}])
63
+ ok!
64
+ end
65
+ end
66
+
67
+ def fail(jid, ex)
68
+ transaction do
69
+ command("FAIL", JSON.dump({ message: ex.message[0...1000],
70
+ errtype: ex.class.name,
71
+ jid: jid,
72
+ backtrace: ex.backtrace}))
73
+ ok!
74
+ end
75
+ end
76
+
77
+ # Sends a heartbeat to the server, in order to prove this
78
+ # worker process is still alive.
79
+ #
80
+ # Return a string signal to process, legal values are "quiet" or "terminate".
81
+ # The quiet signal is informative: the server won't allow this process to FETCH
82
+ # any more jobs anyways.
83
+ def beat
84
+ transaction do
85
+ command("BEAT", %Q[{"wid":"#{@@random_process_wid}"}])
86
+ str = result
87
+ if str == "OK"
88
+ str
89
+ else
90
+ hash = JSON.parse(str)
91
+ hash["signal"]
92
+ end
93
+ end
94
+ end
95
+
96
+ def info
97
+ transaction do
98
+ command("INFO")
99
+ str = result
100
+ JSON.parse(str) if str
101
+ end
102
+ end
103
+
104
+ private
105
+
106
+ def debug(line)
107
+ puts line
108
+ end
109
+
110
+ def tls?
111
+ @location.hostname !~ /\Alocalhost\z/ || @location.scheme =~ /tls/
112
+ end
113
+
114
+ def open
115
+ if tls?
116
+ sock = TCPSocket.new(@location.hostname, @location.port)
117
+ ctx = OpenSSL::SSL::SSLContext.new
118
+ ctx.set_params(verify_mode: OpenSSL::SSL::VERIFY_PEER)
119
+ ctx.ssl_version = :TLSv1_2
120
+
121
+ @sock = OpenSSL::SSL::SSLSocket.new(sock, ctx).tap do |socket|
122
+ socket.sync_close = true
123
+ socket.connect
124
+ end
125
+ else
126
+ @sock = TCPSocket.new(@location.hostname, @location.port)
127
+ @sock.setsockopt(Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, true)
128
+ end
129
+
130
+ payload = {
131
+ "wid": @@random_process_wid,
132
+ "hostname": Socket.gethostname,
133
+ "pid": $$,
134
+ "labels": ["ruby-#{RUBY_VERSION}"],
135
+ }
136
+
137
+ hi = result
138
+
139
+ if hi =~ /\AHI (.*)/
140
+ hash = JSON.parse($1)
141
+ # TODO verify version tag
142
+ salt = hash["s"]
143
+ if salt
144
+ pwd = @location.password
145
+ if !pwd
146
+ raise ArgumentError, "Server requires password, but none has been configured"
147
+ end
148
+ payload["pwdhash"] = Digest::SHA256.hexdigest(pwd + salt)
149
+ end
150
+ end
151
+
152
+ command("HELLO", JSON.dump(payload))
153
+ ok!
154
+ end
155
+
156
+ def command(*args)
157
+ cmd = args.join(" ")
158
+ @sock.puts(cmd)
159
+ debug "> #{cmd}" if @debug
160
+ end
161
+
162
+ def transaction
163
+ retryable = true
164
+ begin
165
+ yield
166
+ rescue Errno::EPIPE, Errno::ECONNRESET
167
+ if retryable
168
+ retryable = false
169
+ open
170
+ retry
171
+ else
172
+ raise
173
+ end
174
+ end
175
+ end
176
+
177
+ # I love pragmatic, simple protocols. Thanks antirez!
178
+ # https://redis.io/topics/protocol
179
+ def result
180
+ line = @sock.gets
181
+ debug "< #{line}" if @debug
182
+ raise Errno::ECONNRESET, "No response" unless line
183
+ chr = line[0]
184
+ if chr == '+'
185
+ line[1..-1].strip
186
+ elsif chr == '$'
187
+ count = line[1..-1].strip.to_i
188
+ data = nil
189
+ data = @sock.read(count) if count > 0
190
+ line = @sock.gets
191
+ data
192
+ elsif chr == '-'
193
+ raise CommandError, line[1..-1]
194
+ else
195
+ # this is bad, indicates we need to reset the socket
196
+ # and start fresh
197
+ raise ParseError, line.strip
198
+ end
199
+ end
200
+
201
+ def ok!
202
+ resp = result
203
+ raise CommandError, resp if resp != "OK"
204
+ true
205
+ end
206
+
207
+ # FAKTORY_PROVIDER=MY_FAKTORY_URL
208
+ # MY_FAKTORY_URL=tcp://:some-pass@some-hostname:7419
209
+ def uri_from_env
210
+ prov = ENV['FAKTORY_PROVIDER']
211
+ return nil unless prov
212
+ raise(ArgumentError, <<-EOM) if prov.index(":")
213
+ Invalid FAKTORY_PROVIDER '#{prov}', it should be the name of the ENV variable that contains the URL
214
+ FAKTORY_PROVIDER=MY_FAKTORY_URL
215
+ MY_FAKTORY_URL=tcp://:some-pass@some-hostname:7419
216
+ EOM
217
+ val = ENV[prov]
218
+ return URI(val) if val
219
+
220
+ val = ENV['FAKTORY_URL']
221
+ return URI(val) if val
222
+ nil
223
+ end
224
+
225
+ end
226
+ end
227
+