spider-gazelle 3.0.4 → 3.0.5

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: be341884962cab440eacc49473a1fe8727e74023
4
- data.tar.gz: fac733a36868410a4733d2720faa79fb1435d3db
3
+ metadata.gz: d98f7022babe544f38425eacb223337cef347ade
4
+ data.tar.gz: d214d5a1192828a1d272f7955817f64f9829c7b9
5
5
  SHA512:
6
- metadata.gz: edf7132a30603414e03d3fca0399851b02bd94dcedb6895ca6f30ca1323901e79ef163ca336b80f521aab52eb705b2e7e15214dd1f297d8a265d88c573668e73
7
- data.tar.gz: e87a60598ea47211402366ca3c6d8fce2bdcb161998720ea7e03f17f311cd22fb3422005bcfd93bc2a45cb9168b3408efc8c752b9d6f6f8b0ce93f99305d4be4
6
+ metadata.gz: 659f91228f5efc1879874407ae54ea20c15df0b9129eb30f8d96b59b57c4979e2da5a4ff0b6686ecd701c4976b36903571156401c93b0c0384b218b08b249d5d
7
+ data.tar.gz: b4264bc65bad3ec4736f25d38b63e0e0ec3e37617b12f794936a9ed66f0081ef114d206856635e6778d991900c7edf9b6ae9d919baef132ff9407510a2ff28cb
@@ -20,14 +20,8 @@ module SpiderGazelle
20
20
  # * Live updates (bindings passed by this pipe)
21
21
  SIGNAL_SERVER = '/tmp/sg-signaller.pipe'
22
22
 
23
- # Spider server is used to
24
- # * Track gazelles
25
- # * Signal shutdown as required
26
- # * Pass sockets
27
- SPIDER_SERVER = '/tmp/sg-spider.pipe.'
28
23
 
29
-
30
- MODES = [:process, :thread, :no_ipc].freeze
24
+ MODES = [:thread, :inline].freeze
31
25
 
32
26
 
33
27
  class LaunchControl
@@ -103,18 +97,6 @@ module SpiderGazelle
103
97
  end
104
98
 
105
99
 
106
-
107
- # ---------------------------------------
108
- # GAZELLE LAUNCH CONTROL
109
- # ---------------------------------------
110
- def start_gazelle(signaller, logger, options)
111
- logger.set_client signaller.pipe
112
-
113
- require 'spider-gazelle/gazelle'
114
- ::SpiderGazelle::Gazelle.new(logger.thread, :process).run!(options)
115
- end
116
-
117
-
118
100
  # ---------------------------------------
119
101
  # TTY SIGNALLING CONTROL
120
102
  # ---------------------------------------
@@ -148,9 +130,6 @@ module SpiderGazelle
148
130
  if master[:spider]
149
131
  logger.verbose "Starting Spider"
150
132
  start_spider(signaller, logger, options)
151
- elsif master[:gazelle]
152
- logger.verbose "Starting Gazelle"
153
- start_gazelle(signaller, logger, options)
154
133
  else
155
134
  logger.verbose "Sending signal to SG Master"
156
135
  signal_master(reactor, signaller, logger, options)
@@ -1,9 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'rack' # Ruby webserver abstraction
4
- require 'spider-gazelle/gazelle/app_store'
5
- require 'spider-gazelle/gazelle/http1'
6
3
 
4
+ require 'rack' # Ruby webserver abstraction
7
5
 
8
6
  # Reactor aware websocket implementation
9
7
  require "spider-gazelle/upgrades/websocket"
@@ -17,149 +15,31 @@ module SpiderGazelle
17
15
  @type = type
18
16
  @logger = Logger.instance
19
17
  @thread = thread
18
+ @thread.ref
19
+ end
20
20
 
21
- @http1_cache = []
22
- @http2_cache = []
23
- @return_http1 = method(:return_http1)
24
- @return_http2 = method(:return_http2)
25
- @parser_count = 0
26
21
 
27
- @on_progress = method(:on_progress)
28
- @set_protocol = method(:set_protocol)
22
+ attr_reader :thread
29
23
 
30
- # Register the gazelle with the signaller so we can shutdown elegantly
31
- if @type == :process
32
- Signaller.instance.gazelle = self
33
- end
34
- end
35
24
 
36
25
  def run!(options)
37
26
  @options = options
38
- @logger.verbose { "Gazelle: #{@type} Pid: #{Process.pid} started" }
39
-
40
- connect_to_spider unless @type == :no_ipc
27
+ @logger.verbose { "Gazelle: #{@type} started" }
41
28
 
42
- load_required_applications
43
29
  self
44
30
  end
45
31
 
46
- def new_app(options)
47
- # TODO:: load this app into all of the gazelles dynamically
48
- end
49
-
50
- def new_connection(data, binding)
51
- socket = @pipe.check_pending
52
- return if socket.nil?
53
- process_connection(socket, data.to_i)
54
- rescue => e
55
- @logger.print_error(e)
56
- end
57
-
58
- def shutdown
59
- # Wait for the requests to finish
60
- @logger.verbose { "Gazelle: #{@type} Pid: #{Process.pid} shutting down" }
61
- end
62
-
63
-
64
- protected
65
-
66
-
67
- def connect_to_spider
68
- @pipe = @thread.pipe :ipc
69
- @pipe.connect(@options[0][:gazelle_ipc]) do |client|
70
- client.progress method(:new_connection)
71
- client.start_read
72
-
73
- authenticate
74
- end
75
-
76
- @pipe.catch do |error, backtrace|
77
- @logger.print_error(error, String.new, backtrace)
78
- end
79
-
80
- if @type == :process
81
- @pipe.finally do
82
- Reactor.instance.shutdown
83
- end
84
- end
85
- end
86
-
87
- def authenticate
88
- @pipe.write "#{@options[0][:gazelle]} #{@type}"
89
- end
90
-
91
- def load_required_applications
92
- @options.each do |app|
93
- if app[:rackup]
94
- AppStore.load(app[:rackup], app)
95
- elsif app[:app]
96
- AppStore.add(app[:app], app)
97
- end
98
- end
99
- end
100
-
101
-
102
- # ---------------------
103
- # Connection Management
104
- # ---------------------
105
- def process_connection(socket, app_id)
106
- # Put application details in the socket storage as we negotiate protocols
107
- details = AppStore.get(app_id)
108
- socket.storage = details
109
- tls = details[-1]
110
-
111
- # Hook up the socket and kick off TLS if required
112
- if tls
113
- socket.on_handshake @set_protocol
114
- socket.start_tls(tls)
115
- else
116
- set_protocol(socket, :http1)
117
- end
118
-
119
- socket.start_read
120
- socket.enable_nodelay
121
- end
122
-
123
- def on_progress(data, socket)
124
- # Storage contains the parser for this connection
125
- parser = socket.storage
126
- parser.parse(data)
127
- end
128
-
129
- def set_protocol(socket, version)
130
- app, port, tls = socket.storage
131
-
132
- parser = if version == :h2
133
- @http2_cache.pop || new_http2_parser
134
- else
135
- @http1_cache.pop || new_http1_parser
32
+ def shutdown(defer)
33
+ @thread.schedule do
34
+ # TODO:: Wait for the requests to finish
35
+ @thread.unref
36
+ @logger.verbose { "Gazelle: #{@type} shutting down" }
37
+ @thread.stop
38
+ defer.resolve(true)
136
39
  end
137
-
138
- parser.load(socket, port, app, tls)
139
- socket.progress @on_progress
140
- socket.storage = parser
141
- end
142
-
143
-
144
- def new_http1_parser
145
- @h1_parser_obj ||= Http1::Callbacks.new
146
-
147
- @parser_count += 1
148
- Http1.new(@return_http1, @h1_parser_obj, @thread, @logger)
149
- end
150
-
151
- def return_http1(parser)
152
- @http1_cache.push parser
153
40
  end
154
41
 
155
- def new_http2_parser
156
- raise NotImplementedError.new 'TODO:: Create HTTP2 parser class'
157
- @parser_count += 1
158
- Http2.new(@return_http2)
159
- end
160
42
 
161
- def return_http2(parser)
162
- @http2_cache << parser
163
- end
43
+ protected
164
44
  end
165
45
  end
@@ -4,97 +4,99 @@ require 'stringio'
4
4
  require 'rack' # Ruby webserver abstraction
5
5
 
6
6
  module SpiderGazelle
7
- class Request < ::Libuv::Q::DeferredPromise
8
-
9
- # TODO:: Add HTTP headers to the env and capitalise them and prefix them with HTTP_
10
- # convert - signs to underscores
11
- PROTO_ENV = {
12
- 'rack.version' => ::Rack::VERSION, # Should be an array of integers
13
- 'rack.errors' => $stderr, # An error stream that supports: puts, write and flush
14
- 'rack.multithread' => true, # can the app be simultaneously invoked by another thread?
15
- 'rack.multiprocess' => false, # will the app be simultaneously be invoked in a separate process?
16
- 'rack.run_once' => false, # this isn't CGI so will always be false
17
- 'SCRIPT_NAME' => ENV['SCRIPT_NAME'] || '', # The virtual path of the app base (empty if root)
18
- 'SERVER_PROTOCOL' => 'HTTP/1.1',
19
-
20
- 'GATEWAY_INTERFACE' => 'CGI/1.2',
21
- 'SERVER_SOFTWARE' => 'SpiderGazelle'
22
- }
23
-
24
- attr_accessor :env, :url, :header, :body, :keep_alive, :upgrade
25
- attr_reader :hijacked, :defer, :is_async
26
-
27
-
28
- def initialize(thread, app, port, remote_ip, scheme, socket)
29
- super(thread, thread.defer)
30
-
31
- @socket = socket
32
- @app = app
33
- @body = String.new
34
- @header = String.new
35
- @url = String.new
36
- @env = PROTO_ENV.dup
37
- @env['SERVER_PORT'] = port
38
- @env['REMOTE_ADDR'] = remote_ip
39
- @env['rack.url_scheme'] = scheme
40
- end
41
-
42
-
43
- def execute!
44
- @env['CONTENT_LENGTH'] = @env.delete('HTTP_CONTENT_LENGTH') || @body.bytesize.to_s
45
- @env['CONTENT_TYPE'] = @env.delete('HTTP_CONTENT_TYPE') || 'text/plain'
46
- @env['REQUEST_URI'] = @url.freeze
47
-
48
- # For Rack::Lint on 1.9, ensure that the encoding is always for spec
49
- @body.force_encoding(Encoding::ASCII_8BIT)
50
- @env['rack.input'] = StringIO.new @body
51
-
52
- # Break the request into its components
53
- query_start = @url.index '?'
54
- if query_start
55
- path = @url[0...query_start].freeze
56
- @env['PATH_INFO'] = path
57
- @env['REQUEST_PATH'] = path
58
- @env['QUERY_STRING'] = @url[query_start + 1..-1].freeze
59
- else
60
- @env['PATH_INFO'] = @url
61
- @env['REQUEST_PATH'] = @url
62
- @env['QUERY_STRING'] = ''
7
+ class Gazelle
8
+ class Request < ::Libuv::Q::DeferredPromise
9
+
10
+ # TODO:: Add HTTP headers to the env and capitalise them and prefix them with HTTP_
11
+ # convert - signs to underscores
12
+ PROTO_ENV = {
13
+ 'rack.version' => ::Rack::VERSION, # Should be an array of integers
14
+ 'rack.errors' => $stderr, # An error stream that supports: puts, write and flush
15
+ 'rack.multithread' => true, # can the app be simultaneously invoked by another thread?
16
+ 'rack.multiprocess' => false, # will the app be simultaneously be invoked in a separate process?
17
+ 'rack.run_once' => false, # this isn't CGI so will always be false
18
+ 'SCRIPT_NAME' => ENV['SCRIPT_NAME'] || '', # The virtual path of the app base (empty if root)
19
+ 'SERVER_PROTOCOL' => 'HTTP/1.1',
20
+
21
+ 'GATEWAY_INTERFACE' => 'CGI/1.2',
22
+ 'SERVER_SOFTWARE' => 'SpiderGazelle'
23
+ }
24
+
25
+ attr_accessor :env, :url, :header, :body, :keep_alive, :upgrade
26
+ attr_reader :hijacked, :defer, :is_async
27
+
28
+
29
+ def initialize(thread, app, port, remote_ip, scheme, socket)
30
+ super(thread, thread.defer)
31
+
32
+ @socket = socket
33
+ @app = app
34
+ @body = String.new
35
+ @header = String.new
36
+ @url = String.new
37
+ @env = PROTO_ENV.dup
38
+ @env['SERVER_PORT'] = port
39
+ @env['REMOTE_ADDR'] = remote_ip
40
+ @env['rack.url_scheme'] = scheme
63
41
  end
64
-
65
- # Grab the host name from the request
66
- if host = @env['HTTP_HOST']
67
- if colon = host.index(':')
68
- @env['SERVER_NAME'] = host[0, colon]
69
- @env['SERVER_PORT'] = host[colon + 1, host.bytesize]
42
+
43
+
44
+ def execute!
45
+ @env['CONTENT_LENGTH'] = @env.delete('HTTP_CONTENT_LENGTH') || @body.bytesize.to_s
46
+ @env['CONTENT_TYPE'] = @env.delete('HTTP_CONTENT_TYPE') || 'text/plain'
47
+ @env['REQUEST_URI'] = @url.freeze
48
+
49
+ # For Rack::Lint on 1.9, ensure that the encoding is always for spec
50
+ @body.force_encoding(Encoding::ASCII_8BIT)
51
+ @env['rack.input'] = StringIO.new @body
52
+
53
+ # Break the request into its components
54
+ query_start = @url.index '?'
55
+ if query_start
56
+ path = @url[0...query_start].freeze
57
+ @env['PATH_INFO'] = path
58
+ @env['REQUEST_PATH'] = path
59
+ @env['QUERY_STRING'] = @url[query_start + 1..-1].freeze
70
60
  else
71
- @env['SERVER_NAME'] = host
61
+ @env['PATH_INFO'] = @url
62
+ @env['REQUEST_PATH'] = @url
63
+ @env['QUERY_STRING'] = ''
72
64
  end
73
- else
74
- @env['SERVER_NAME'] = 'localhost'
75
- end
76
-
77
- if @upgrade == true && @env['HTTP_UPGRADE'] == 'h2c'
78
- # TODO:: implement the upgrade process here
79
- end
80
65
 
81
- # Provide hijack options
82
- @env['rack.hijack?'] = true
83
- @env['rack.hijack'] = proc { @env['rack.hijack_io'] = @socket }
66
+ # Grab the host name from the request
67
+ if host = @env['HTTP_HOST']
68
+ if colon = host.index(':')
69
+ @env['SERVER_NAME'] = host[0, colon]
70
+ @env['SERVER_PORT'] = host[colon + 1, host.bytesize]
71
+ else
72
+ @env['SERVER_NAME'] = host
73
+ end
74
+ else
75
+ @env['SERVER_NAME'] = 'localhost'
76
+ end
84
77
 
85
- # Execute the request
86
- # NOTE:: Catch was overloaded by Promise so this does the trick now
87
- resp = ruby_catch(:async) { @app.call @env }
88
- if resp.nil? || resp[0] == -1
89
- @is_async = true
78
+ if @upgrade == true && @env['HTTP_UPGRADE'] == 'h2c'
79
+ # TODO:: implement the upgrade process here
80
+ end
90
81
 
91
- # close the body for deferred responses
92
- unless resp.nil?
93
- body = resp[2]
94
- body.close if body.respond_to?(:close)
82
+ # Provide hijack options
83
+ @env['rack.hijack?'] = true
84
+ @env['rack.hijack'] = proc { @env['rack.hijack_io'] = @socket }
85
+
86
+ # Execute the request
87
+ # NOTE:: Catch was overloaded by Promise so this does the trick now
88
+ resp = ruby_catch(:async) { @app.call @env }
89
+ if resp.nil? || resp[0] == -1
90
+ @is_async = true
91
+
92
+ # close the body for deferred responses
93
+ unless resp.nil?
94
+ body = resp[2]
95
+ body.close if body.respond_to?(:close)
96
+ end
95
97
  end
98
+ resp
96
99
  end
97
- resp
98
100
  end
99
101
  end
100
102
  end
@@ -64,7 +64,7 @@ module SpiderGazelle
64
64
  options[:lint] = true
65
65
  end
66
66
 
67
- opts.on "-m", "--mode MODE", MODES, "Either process, thread or no_ipc (default: thread)" do |arg|
67
+ opts.on "-m", "--mode MODE", MODES, "Either reactor, thread or inline (default: reactor)" do |arg|
68
68
  options[:mode] = arg
69
69
  end
70
70
 
@@ -1,6 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'spider-gazelle/spider/app_store' # Manages the loaded applications
3
4
  require 'spider-gazelle/spider/binding' # Holds a reference to a bound port
5
+ require 'spider-gazelle/spider/http1' # Parses and responds to HTTP1 requests
4
6
  require 'securerandom'
5
7
 
6
8
 
@@ -30,9 +32,8 @@ module SpiderGazelle
30
32
 
31
33
  # Gazelle pipe connection management
32
34
  @gazelles = {
33
- # process: [],
34
35
  # thread: [],
35
- # no_ipc: gazelle_instance
36
+ # inline: gazelle_instance
36
37
  }
37
38
  @counts = {
38
39
  # process: number
@@ -41,9 +42,7 @@ module SpiderGazelle
41
42
  @loading = {} # mode => load defer
42
43
  @bindings = {} # port => binding
43
44
  @iterators = {} # mode => gazelle round robin iterator
44
- @iterator_source = {} # mode => gazelle pipe array (iterator source)
45
-
46
- @password = SecureRandom.hex
45
+ @iterator_source = {} # mode => gazelle thread array (iterator source)
47
46
 
48
47
  @running = true
49
48
  @loaded = false
@@ -64,7 +63,6 @@ module SpiderGazelle
64
63
 
65
64
  # Load gazelles and make the required bindings
66
65
  def ready
67
- start_gazelle_server
68
66
  load_promise = load_applications
69
67
  load_promise.then do
70
68
  # Check a shutdown request didn't occur as we were loading
@@ -116,82 +114,6 @@ module SpiderGazelle
116
114
  protected
117
115
 
118
116
 
119
- # This starts the server the gazelles will be connecting to
120
- def start_gazelle_server
121
- @pipe_file = "#{SPIDER_SERVER}#{Process.pid}"
122
- @logger.verbose { "Spider server starting on #{@pipe_file}" }
123
-
124
- @pipe = @thread.pipe :ipc
125
- begin
126
- File.unlink @pipe_file
127
- rescue
128
- end
129
-
130
- @shutdown = false
131
- check = method(:check_credentials)
132
- @pipe.bind(@pipe_file) do |client|
133
- @logger.verbose { "Gazelle <0x#{client.object_id.to_s(16)}> connection made" }
134
-
135
- # Shutdown if there is an error with any of the gazelles
136
- client.catch do |error|
137
- begin
138
- @logger.print_error(error, "Gazelle <0x#{client.object_id.to_s(16)}> connection error")
139
- rescue
140
- end
141
- end
142
-
143
- # Client closed gracefully
144
- client.finally do
145
- begin
146
- @logger.verbose { "Gazelle <0x#{client.object_id.to_s(16)}> disconnected" }
147
- rescue
148
- ensure
149
- @gazelles.delete client
150
- if !@shutdown
151
- @shutdown = true
152
- @signaller.general_failure
153
- end
154
- end
155
- end
156
-
157
- client.progress check
158
- client.start_read
159
- end
160
-
161
- # catch server errors
162
- @pipe.catch do |error|
163
- @logger.print_error(error)
164
- @signaller.general_failure
165
- end
166
-
167
- # start listening
168
- @pipe.listen(INTERNAL_PIPE_BACKLOG)
169
- end
170
-
171
- def check_credentials(data, gazelle)
172
- password, mode = data.split(' ', 2)
173
- mode_sym = mode.to_sym
174
-
175
- if password == @password && MODES.include?(mode_sym)
176
- @gazelles[mode_sym] ||= []
177
- gazelles = @gazelles[mode_sym]
178
- gazelles << gazelle
179
- @logger.verbose { "Gazelle <0x#{gazelle.object_id.to_s(16)}> connection was validated" }
180
-
181
- # All the gazelles have loaded. Lets start processing requests
182
- if gazelles.length == @counts[mode_sym]
183
- @logger.verbose { "#{mode.capitalize} gazelles are ready" }
184
-
185
- @iterator_source[mode_sym] = gazelles
186
- @iterators[mode_sym] = gazelles.cycle
187
- @loading[mode_sym].resolve(true)
188
- end
189
- else
190
- @logger.warn "Gazelle <0x#{gazelle.object_id.to_s(16)}> connection closed due to bad credentials"
191
- gazelle.close
192
- end
193
- end
194
-
195
117
  def load_applications
196
118
  loaded = []
197
119
  @logger.info "Environment: #{ENV['RACK_ENV']} on #{RUBY_ENGINE || 'ruby'} #{RUBY_VERSION}"
@@ -199,6 +121,7 @@ module SpiderGazelle
199
121
  # Load the different types of gazelles required
200
122
  @options.each do |app|
201
123
  @logger.info "Loading: #{app[:rackup]}" if app[:rackup]
124
+ AppStore.load(app[:rackup], app)
202
125
 
203
126
  mode = app[:mode]
204
127
  loaded << load_gazelles(mode, app[:count], @options) unless @loading[mode]
@@ -209,65 +132,54 @@ module SpiderGazelle
209
132
  @thread.all(*loaded)
210
133
  end
211
134
 
212
-
135
+
213
136
  def load_gazelles(mode, count, options)
214
137
  defer = @thread.defer
215
138
  @loading[mode] = defer
216
139
 
217
- pass = options[0][:spider]
218
-
219
- if mode == :no_ipc
220
- # Provide the password to the gazelle instance
221
- options[0][:gazelle] = @password
222
-
140
+ if mode == :inline
223
141
  # Start the gazelle
224
142
  require 'spider-gazelle/gazelle'
225
143
  gaz = ::SpiderGazelle::Gazelle.new(@thread, mode).run!(options)
226
- @gazelles[:no_ipc] = gaz
144
+ @gazelles[:inline] = gaz
227
145
 
228
146
  # Setup the round robin
229
- @iterator_source[mode] = gaz
230
- @iterators[mode] = gaz
147
+ itr = [gaz]
148
+ @iterator_source[mode] = [gaz]
149
+ @iterators[mode] = [gaz.thread].cycle
150
+
231
151
  defer.resolve(true)
232
152
  else
233
153
  require 'thread'
234
154
 
235
- # Provide the password to the gazelle instance
236
- options[0][:gazelle] = @password
237
- options[0][:gazelle_ipc] = @pipe_file
238
-
239
155
  count = @counts[mode] = count || ::Libuv.cpu_count || 1
240
156
  @logger.info "#{mode.to_s.capitalize} count: #{count}"
241
157
 
242
- if mode == :thread
243
- require 'spider-gazelle/gazelle'
244
- reactor = Reactor.instance
245
-
246
- @threads = []
247
- count.times do
248
- thread = ::Libuv::Reactor.new
249
- @threads << thread
158
+ require 'spider-gazelle/gazelle'
159
+ reactor = Reactor.instance
250
160
 
251
- Thread.new { load_gazelle_thread(reactor, thread, mode, options) }
252
- end
253
- else
254
- # Remove the spider option
255
- args = LaunchControl.instance.args - ['-s', pass]
161
+ @threads = []
162
+ loaded = []
163
+ count.times do
164
+ loading = @thread.defer
165
+ loaded << loading.promise
256
166
 
257
- # Build the command with the gazelle option
258
- args = [EXEC_NAME, '-g', @password, '-f', @pipe_file] + args
167
+ thread = ::Libuv::Reactor.new
168
+ @threads << thread
259
169
 
260
- @logger.verbose { "Launching #{count} gazelle processes" }
261
- count.times do
262
- Thread.new { launch_gazelle(args) }
263
- end
170
+ Thread.new { load_gazelle_thread(reactor, thread, mode, options, loading) }
264
171
  end
172
+
173
+ defer.resolve(@thread.all(*loaded).then { |gazelles|
174
+ @iterator_source[mode] = gazelles
175
+ @iterators[mode] = gazelles.map { |gaz| gaz.thread }.cycle
176
+ })
265
177
  end
266
178
 
267
179
  defer.promise
268
180
  end
269
181
 
270
- def load_gazelle_thread(reactor, thread, mode, options)
182
+ def load_gazelle_thread(reactor, thread, mode, options, loading)
271
183
  # Log any unhandled errors
272
184
  thread.notifier reactor.method(:log)
273
185
  # Give current requests 5 seconds to complete
@@ -281,25 +193,11 @@ module SpiderGazelle
281
193
  end
282
194
  thread.run do |thread|
283
195
  # Start the gazelle
284
- ::SpiderGazelle::Gazelle.new(thread, :thread).run!(options)
285
- end
286
- end
287
-
288
- def launch_gazelle(cmd)
289
- # Wait for the process to close
290
- result = system(*cmd)
291
-
292
- # Kill the spider if a process exits unexpectedly
293
- if @running
294
- @thread.schedule do
295
- if result
296
- @logger.verbose "Gazelle process exited with exit status 0"
297
- else
298
- @logger.error "Gazelle process exited unexpectedly"
299
- end
300
-
301
- @signaller.general_failure
196
+ gaz = ::SpiderGazelle::Gazelle.new(thread, :thread)
197
+ thread.next_tick do
198
+ loading.resolve(gaz)
302
199
  end
200
+ gaz.run!(options)
303
201
  end
304
202
  end
305
203
 
@@ -307,12 +205,11 @@ module SpiderGazelle
307
205
  @bound = true
308
206
  @loaded = true
309
207
 
310
- @options.each_index do |id|
311
- options = @options[id]
208
+ @options.each do |options|
312
209
  @logger.verbose { "Loading rackup #{options}" }
313
210
  iterator = @iterators[options[:mode]]
314
211
 
315
- binding = @bindings[options[:port]] = Binding.new(iterator, id.to_s, options)
212
+ binding = @bindings[options[:port]] = Binding.new(iterator, options)
316
213
  binding.bind
317
214
  end
318
215
  end
@@ -344,17 +241,11 @@ module SpiderGazelle
344
241
  promises = []
345
242
 
346
243
  @iterator_source.each do |mode, gazelles|
347
- if mode == :no_ipc
348
- # itr is a gazelle in no_ipc mode
244
+ # End communication with the gazelle threads
245
+ gazelles.each do |gazelle|
349
246
  defer = @thread.defer
350
- gazelles.shutdown(defer)
247
+ gazelle.shutdown(defer)
351
248
  promises << defer.promise
352
-
353
- else
354
- # End communication with the gazelle threads / processes
355
- gazelles.dup.each do |gazelle|
356
- promises << gazelle.close
357
- end
358
249
  end
359
250
  end
360
251
 
@@ -3,7 +3,7 @@
3
3
  require 'thread'
4
4
 
5
5
  module SpiderGazelle
6
- class Gazelle
6
+ class Spider
7
7
  module AppStore
8
8
  @apps = []
9
9
  @loaded = {}
@@ -21,7 +21,7 @@ module SpiderGazelle
21
21
  tls = configure_tls(options)
22
22
  port = tls ? 443 : 80
23
23
 
24
- val = [app, port.to_s, tls]
24
+ val = [app, port, tls]
25
25
  @apps << val
26
26
  @loaded[rackup] = val
27
27
  }
@@ -5,36 +5,45 @@ require 'thread'
5
5
  module SpiderGazelle
6
6
  class Spider
7
7
  class Binding
8
- attr_reader :app_id
9
- attr_accessor :tcp
8
+ attr_reader :tcp, :app, :app_port, :tls
10
9
 
11
10
 
12
- def initialize(iterator, app_id, options)
13
- if options[:mode] == :no_ipc
14
- @delegate = method(:direct_delegate)
15
- @gazelle = iterator
16
- else
17
- @delegate = method(:delegate)
18
- @select_gazelle = iterator
19
- end
20
-
11
+ def initialize(iterator, options)
21
12
  @options = options
22
13
 
23
14
  @logger = Logger.instance
24
15
  @signaller = Signaller.instance
25
16
  @thread = @signaller.thread
17
+ @iterator = iterator
26
18
 
27
19
  @port = @options[:port]
28
- @indicator = app_id.to_s.freeze
20
+ @app, @app_port, @tls = AppStore.lookup(options[:rackup])
21
+
22
+ @set_protocol = method(:set_protocol)
23
+
24
+ @http1_cache = []
25
+ @http2_cache = []
26
+ @return_http1 = method(:return_http1)
27
+ @return_http2 = method(:return_http2)
28
+ @parser_count = 0
29
29
  end
30
30
 
31
31
  # Bind the application to the selected port
32
32
  def bind
33
33
  # Bind the socket
34
34
  @tcp = @thread.tcp
35
- @tcp.bind @options[:host], @port, @delegate
36
- @tcp.listen @options[:backlog]
37
35
  @tcp.enable_simultaneous_accepts
36
+
37
+ if @tls
38
+ @tcp.bind @options[:host], @port do |client|
39
+ prepare_client_tls client
40
+ end
41
+ else
42
+ @tcp.bind @options[:host], @port do |client|
43
+ prepare_client client
44
+ end
45
+ end
46
+ @tcp.listen 10000
38
47
 
39
48
  @logger.info "Listening on tcp://#{@options[:host]}:#{@port}"
40
49
 
@@ -59,23 +68,59 @@ module SpiderGazelle
59
68
  protected
60
69
 
61
70
 
62
- DELEGATE_ERR = proc { |error|
63
- client.close
64
- begin
65
- Logger.instance.print_error(error, "delegating socket to gazelle")
66
- rescue
71
+ # ---------------------------
72
+ # Setup the client connection
73
+ # ---------------------------
74
+ def prepare_client(client)
75
+ set_protocol(client, :http1)
76
+ client.start_read
77
+ client.enable_nodelay
78
+ end
79
+
80
+ def prepare_client_tls(client)
81
+ client.on_handshake @set_protocol
82
+ client.start_tls(@tls)
83
+
84
+ client.start_read
85
+ client.enable_nodelay
86
+ end
87
+
88
+ def set_protocol(client, version)
89
+ parser = if version == :h2
90
+ @http2_cache.pop || new_http2_parser
91
+ else
92
+ @http1_cache.pop || new_http1_parser
67
93
  end
68
- }
69
- def delegate(client, retries = 0)
70
- promise = @select_gazelle.next.write2(client, @indicator, wait: :promise)
71
- promise.then do
72
- client.close
94
+
95
+ parser.load(client, @app_port, @app, @tls)
96
+ client.progress do |data, _|
97
+ parser.parse(data)
73
98
  end
74
- promise.catch DELEGATE_ERR
75
99
  end
76
100
 
77
- def direct_delegate(client)
78
- @gazelle.__send__(:process_connection, client, @indicator)
101
+
102
+ # ---------------------------------------
103
+ # Select a protocol to handle the request
104
+ # ---------------------------------------
105
+ def new_http1_parser
106
+ @h1_parser_obj ||= Http1::Callbacks.new
107
+
108
+ @parser_count += 1
109
+ Http1.new(@return_http1, @h1_parser_obj, @thread, @logger, @iterator)
110
+ end
111
+
112
+ def return_http1(parser)
113
+ @http1_cache.push parser
114
+ end
115
+
116
+ def new_http2_parser
117
+ raise NotImplementedError.new 'TODO:: Create HTTP2 parser class'
118
+ @parser_count += 1
119
+ Http2.new(@return_http2, @thread, @logger, @iterator)
120
+ end
121
+
122
+ def return_http2(parser)
123
+ @http2_cache << parser
79
124
  end
80
125
  end
81
126
  end
@@ -5,7 +5,7 @@ require 'spider-gazelle/gazelle/request'
5
5
 
6
6
 
7
7
  module SpiderGazelle
8
- class Gazelle
8
+ class Spider
9
9
  class Http1
10
10
  class Callbacks
11
11
  def initialize
@@ -67,7 +67,7 @@ module SpiderGazelle
67
67
  Hijack = Struct.new :socket, :env
68
68
 
69
69
 
70
- def initialize(return_method, callbacks, thread, logger)
70
+ def initialize(return_method, callbacks, thread, logger, gazelles)
71
71
  # The HTTP parser callbacks object for this thread
72
72
  @return_method = return_method
73
73
  @callbacks = callbacks
@@ -76,6 +76,7 @@ module SpiderGazelle
76
76
 
77
77
  @queue_response = method :queue_response
78
78
  @write_chunk = method :write_chunk
79
+ @gazelles = gazelles
79
80
 
80
81
  # The parser state for this instance
81
82
  @state = ::HttpParser::Parser.new_instance do |inst|
@@ -150,7 +151,7 @@ module SpiderGazelle
150
151
  # Parser Callbacks
151
152
  # ----------------
152
153
  def start_parsing
153
- @parsing = Request.new @thread, @app, @port, @remote_ip, @scheme, @socket
154
+ @parsing = Gazelle::Request.new @thread, @app, @port, @remote_ip, @scheme, @socket
154
155
  end
155
156
 
156
157
  def headers_complete
@@ -160,12 +161,14 @@ module SpiderGazelle
160
161
  def finished_parsing
161
162
  request = @parsing
162
163
  @parsing = nil
164
+ request.keep_alive = @state.keep_alive?
165
+ request.upgrade = @state.upgrade?
163
166
 
164
- if !@state.keep_alive?
165
- request.keep_alive = false
166
- # We don't expect any more data
167
- @socket.stop_read
168
- end
167
+ @thread.next_tick { after_parsing(request) }
168
+ end
169
+
170
+ def after_parsing(request)
171
+ @socket.stop_read unless request.keep_alive
169
172
 
170
173
  # Process the async request in the same way as Mizuno
171
174
  # See: http://polycrystal.org/2012/04/15/asynchronous_responses_in_rack.html
@@ -173,12 +176,9 @@ module SpiderGazelle
173
176
  request.env['async.callback'] = proc { |data|
174
177
  @thread.schedule { request.defer.resolve([request, data]) }
175
178
  }
176
- request.upgrade = @state.upgrade?
177
179
  @requests << request
178
180
 
179
- unless @processing
180
- ::Fiber.new { process_next }.resume
181
- end
181
+ process_next unless @processing
182
182
  end
183
183
 
184
184
  # ------------------
@@ -189,29 +189,34 @@ module SpiderGazelle
189
189
  @processing = @requests.shift
190
190
  if @processing
191
191
  request = @processing
192
- begin
193
- result = begin
194
- request.execute!
195
- rescue StandardError => e
196
- @logger.print_error e, 'framework error'
197
- @processing.keep_alive = false
198
- [500, {}, EMPTY_RESPONSE]
199
- end
192
+ request.then @queue_response
200
193
 
201
- if request.is_async && !request.hijacked
202
- if result.nil? && !request.defer.resolved?
203
- # TODO:: setup timeout for async response
204
- end
205
- else
206
- # Complete the current request
207
- request.defer.resolve([request, result])
208
- end
209
- request.then @queue_response
210
- rescue Exception => error
211
- Logger.instance.print_error error, 'critical error'
212
- Reactor.instance.shutdown
194
+ @gazelles.next.schedule do
195
+ process_on_gazelle(request)
196
+ end
197
+ end
198
+ end
199
+
200
+ def process_on_gazelle(request)
201
+ result = begin
202
+ request.execute!
203
+ rescue StandardError => e
204
+ Logger.instance.print_error e, 'framework error'
205
+ request.keep_alive = false
206
+ [500, {}, EMPTY_RESPONSE]
207
+ end
208
+
209
+ if request.is_async && !request.hijacked
210
+ if result.nil? && !request.defer.resolved?
211
+ # TODO:: setup timeout for async response
213
212
  end
213
+ else
214
+ # Complete the current request
215
+ request.defer.resolve([request, result])
214
216
  end
217
+ rescue Exception => error
218
+ Logger.instance.print_error error, 'critical error'
219
+ Reactor.instance.shutdown
215
220
  end
216
221
 
217
222
  # ----------------
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module SpiderGazelle
4
- VERSION = '3.0.4'
4
+ VERSION = '3.0.5'
5
5
  EXEC_NAME = 'sg'
6
6
  end
data/spec/http1_spec.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  require 'spider-gazelle'
2
- require 'spider-gazelle/gazelle/http1'
2
+ require 'spider-gazelle/spider/http1'
3
3
 
4
4
 
5
5
  # TODO:: Mock logger
@@ -50,7 +50,7 @@ class MockLogger
50
50
  end
51
51
  end
52
52
 
53
- describe ::SpiderGazelle::Gazelle::Http1 do
53
+ describe ::SpiderGazelle::Spider::Http1 do
54
54
  before :each do
55
55
  @shutdown_called = 0
56
56
  @close_called = 0
@@ -73,8 +73,8 @@ describe ::SpiderGazelle::Gazelle::Http1 do
73
73
  @returned = http1
74
74
  }
75
75
  @logger = MockLogger.new
76
- @http1_callbacks ||= ::SpiderGazelle::Gazelle::Http1::Callbacks.new
77
- @http1 = ::SpiderGazelle::Gazelle::Http1.new(@return, @http1_callbacks, @loop, @logger)
76
+ @http1_callbacks ||= ::SpiderGazelle::Spider::Http1::Callbacks.new
77
+ @http1 = ::SpiderGazelle::Spider::Http1.new(@return, @http1_callbacks, @loop, @logger, [@loop].cycle)
78
78
 
79
79
  @socket = MockSocket.new
80
80
  @socket.shutdown_cb = proc {
@@ -26,7 +26,7 @@ Gem::Specification.new do |s|
26
26
  s.add_dependency 'http-2', '~> 0.8' # HTTP2 parsing and response management
27
27
 
28
28
  s.add_development_dependency 'rspec', '~> 3.5' # Testing framework
29
- s.add_development_dependency 'rake', '~> 11.2' # Task runner
29
+ s.add_development_dependency 'rake', '~> 12' # Task runner
30
30
  s.add_development_dependency 'yard', '~> 0.9' # Comment based documentation generation
31
31
 
32
32
  s.files = Dir["{lib,bin}/**/*"] + %w(Rakefile spider-gazelle.gemspec README.md LICENSE)
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: spider-gazelle
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.0.4
4
+ version: 3.0.5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Stephen von Takach
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-03-19 00:00:00.000000000 Z
11
+ date: 2017-03-27 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: http-parser
@@ -114,14 +114,14 @@ dependencies:
114
114
  requirements:
115
115
  - - "~>"
116
116
  - !ruby/object:Gem::Version
117
- version: '11.2'
117
+ version: '12'
118
118
  type: :development
119
119
  prerelease: false
120
120
  version_requirements: !ruby/object:Gem::Requirement
121
121
  requirements:
122
122
  - - "~>"
123
123
  - !ruby/object:Gem::Version
124
- version: '11.2'
124
+ version: '12'
125
125
  - !ruby/object:Gem::Dependency
126
126
  name: yard
127
127
  requirement: !ruby/object:Gem::Requirement
@@ -155,8 +155,6 @@ files:
155
155
  - lib/rack/handler/spider-gazelle.rb
156
156
  - lib/spider-gazelle.rb
157
157
  - lib/spider-gazelle/gazelle.rb
158
- - lib/spider-gazelle/gazelle/app_store.rb
159
- - lib/spider-gazelle/gazelle/http1.rb
160
158
  - lib/spider-gazelle/gazelle/request.rb
161
159
  - lib/spider-gazelle/logger.rb
162
160
  - lib/spider-gazelle/options.rb
@@ -164,7 +162,9 @@ files:
164
162
  - lib/spider-gazelle/signaller.rb
165
163
  - lib/spider-gazelle/signaller/signal_parser.rb
166
164
  - lib/spider-gazelle/spider.rb
165
+ - lib/spider-gazelle/spider/app_store.rb
167
166
  - lib/spider-gazelle/spider/binding.rb
167
+ - lib/spider-gazelle/spider/http1.rb
168
168
  - lib/spider-gazelle/upgrades/websocket.rb
169
169
  - lib/spider-gazelle/version.rb
170
170
  - spec/http1_spec.rb