hyperion-rb 1.0.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +44 -0
- data/README.md +32 -1
- data/ext/hyperion_http/parser.c +141 -0
- data/lib/hyperion/adapter/rack.rb +14 -0
- data/lib/hyperion/admin_middleware.rb +110 -0
- data/lib/hyperion/cli.rb +82 -1
- data/lib/hyperion/config.rb +11 -1
- data/lib/hyperion/connection.rb +56 -4
- data/lib/hyperion/http2_handler.rb +243 -6
- data/lib/hyperion/logger.rb +94 -3
- data/lib/hyperion/master.rb +69 -1
- data/lib/hyperion/prometheus_exporter.rb +96 -0
- data/lib/hyperion/response_writer.rb +87 -10
- data/lib/hyperion/server.rb +106 -32
- data/lib/hyperion/thread_pool.rb +24 -8
- data/lib/hyperion/version.rb +1 -1
- data/lib/hyperion/worker.rb +19 -11
- data/lib/hyperion/worker_health.rb +33 -0
- data/lib/hyperion.rb +58 -0
- metadata +4 -1
data/lib/hyperion/connection.rb
CHANGED
|
@@ -17,6 +17,7 @@ module Hyperion
|
|
|
17
17
|
MAX_BODY_BYTES = 16 * 1024 * 1024 # 16 MB cap. Phase 5 introduces streaming bodies.
|
|
18
18
|
HEADER_TERM = "\r\n\r\n"
|
|
19
19
|
TIMEOUT_SENTINEL = :__hyperion_read_timeout__
|
|
20
|
+
DEADLINE_SENTINEL = :__hyperion_request_deadline__
|
|
20
21
|
IDLE_KEEPALIVE_TIMEOUT_SECONDS = 5
|
|
21
22
|
|
|
22
23
|
# Default parser is the C-extension `CParser` when the extension built;
|
|
@@ -44,14 +45,20 @@ module Hyperion
|
|
|
44
45
|
@log_requests = log_requests.nil? ? Hyperion.log_requests? : log_requests
|
|
45
46
|
end
|
|
46
47
|
|
|
47
|
-
def serve(socket, app)
|
|
48
|
+
def serve(socket, app, max_request_read_seconds: 60)
|
|
48
49
|
request_count = 0
|
|
49
50
|
carry = +'' # bytes already pulled off the socket but past the prev request boundary
|
|
50
51
|
peer_addr = peer_address(socket)
|
|
51
52
|
@metrics.increment(:connections_accepted)
|
|
52
53
|
@metrics.increment(:connections_active)
|
|
53
54
|
loop do
|
|
54
|
-
|
|
55
|
+
# Per-request wallclock deadline. Captured fresh for every request so
|
|
56
|
+
# long-lived keep-alive sessions with many small requests don't
|
|
57
|
+
# falsely trip after the cumulative budget elapses.
|
|
58
|
+
request_started_clock = Process.clock_gettime(Process::CLOCK_MONOTONIC) if max_request_read_seconds
|
|
59
|
+
buffer = read_request(socket, carry, deadline_started_at: request_started_clock,
|
|
60
|
+
max_request_read_seconds: max_request_read_seconds,
|
|
61
|
+
peer_addr: peer_addr)
|
|
55
62
|
return unless buffer
|
|
56
63
|
|
|
57
64
|
if buffer == TIMEOUT_SENTINEL
|
|
@@ -65,10 +72,15 @@ module Hyperion
|
|
|
65
72
|
return
|
|
66
73
|
end
|
|
67
74
|
|
|
75
|
+
# Slowloris-style abort: deadline tripped during read. We've already
|
|
76
|
+
# written the 408 (best-effort) inside read_request; close out here.
|
|
77
|
+
return if buffer == DEADLINE_SENTINEL
|
|
78
|
+
|
|
68
79
|
request, body_end = @parser.parse(buffer)
|
|
69
80
|
carry = +(buffer.byteslice(body_end, buffer.bytesize - body_end) || '')
|
|
70
81
|
request = enrich_with_peer(request, peer_addr) if peer_addr && request.peer_address.nil?
|
|
71
82
|
|
|
83
|
+
@metrics.increment(:bytes_read, body_end)
|
|
72
84
|
@metrics.increment(:requests_total)
|
|
73
85
|
@metrics.increment(:requests_in_flight)
|
|
74
86
|
request_started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC) if @log_requests
|
|
@@ -192,10 +204,16 @@ module Hyperion
|
|
|
192
204
|
# pipelining). Returns the full buffer (with any trailing pipelined
|
|
193
205
|
# bytes intact); the parser's returned end_offset tells the caller
|
|
194
206
|
# where this request ends. On EOF returns nil; on read timeout returns
|
|
195
|
-
# TIMEOUT_SENTINEL
|
|
196
|
-
|
|
207
|
+
# TIMEOUT_SENTINEL; on per-request wallclock deadline trip returns
|
|
208
|
+
# DEADLINE_SENTINEL (and emits a best-effort 408 + close).
|
|
209
|
+
def read_request(socket, carry = +'', deadline_started_at: nil, max_request_read_seconds: nil,
|
|
210
|
+
peer_addr: nil)
|
|
197
211
|
buffer = carry
|
|
198
212
|
until buffer.include?(HEADER_TERM)
|
|
213
|
+
if deadline_exceeded?(deadline_started_at, max_request_read_seconds)
|
|
214
|
+
return abort_for_deadline(socket, deadline_started_at, peer_addr)
|
|
215
|
+
end
|
|
216
|
+
|
|
199
217
|
chunk = read_chunk(socket)
|
|
200
218
|
return chunk if chunk.nil? || chunk == TIMEOUT_SENTINEL
|
|
201
219
|
return nil if chunk.empty?
|
|
@@ -210,6 +228,9 @@ module Hyperion
|
|
|
210
228
|
if chunked?(headers_part)
|
|
211
229
|
until chunked_body_complete?(buffer, header_end)
|
|
212
230
|
raise ParseError, 'chunked body exceeds limit' if buffer.bytesize - header_end > MAX_BODY_BYTES
|
|
231
|
+
if deadline_exceeded?(deadline_started_at, max_request_read_seconds)
|
|
232
|
+
return abort_for_deadline(socket, deadline_started_at, peer_addr)
|
|
233
|
+
end
|
|
213
234
|
|
|
214
235
|
chunk = read_chunk(socket)
|
|
215
236
|
break if chunk.nil? || chunk.empty? || chunk == TIMEOUT_SENTINEL
|
|
@@ -219,6 +240,10 @@ module Hyperion
|
|
|
219
240
|
else
|
|
220
241
|
content_length = headers_part[/^content-length:\s*(\d+)/i, 1].to_i
|
|
221
242
|
while buffer.bytesize < header_end + content_length
|
|
243
|
+
if deadline_exceeded?(deadline_started_at, max_request_read_seconds)
|
|
244
|
+
return abort_for_deadline(socket, deadline_started_at, peer_addr)
|
|
245
|
+
end
|
|
246
|
+
|
|
222
247
|
chunk = read_chunk(socket)
|
|
223
248
|
break if chunk.nil? || chunk.empty? || chunk == TIMEOUT_SENTINEL
|
|
224
249
|
|
|
@@ -229,6 +254,33 @@ module Hyperion
|
|
|
229
254
|
buffer
|
|
230
255
|
end
|
|
231
256
|
|
|
257
|
+
# nil-disabled or budget-untripped → false. Otherwise the wallclock cap
|
|
258
|
+
# has been exceeded and the caller should abort.
|
|
259
|
+
def deadline_exceeded?(started_at, max_seconds)
|
|
260
|
+
return false unless started_at && max_seconds
|
|
261
|
+
|
|
262
|
+
(Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at) > max_seconds
|
|
263
|
+
end
|
|
264
|
+
|
|
265
|
+
# Slowloris fallback: log a structured warn, bump :slow_request_aborts,
|
|
266
|
+
# write a best-effort 408, and let the caller close the socket. We don't
|
|
267
|
+
# wait on the 408 write — a dribbling client may never read it, and
|
|
268
|
+
# that's the failure mode we're protecting against anyway.
|
|
269
|
+
def abort_for_deadline(socket, started_at, peer_addr)
|
|
270
|
+
elapsed = started_at ? (Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at).round(3) : nil
|
|
271
|
+
@metrics.increment(:slow_request_aborts)
|
|
272
|
+
@logger.warn do
|
|
273
|
+
{ message: 'request read deadline exceeded', remote_addr: peer_addr, elapsed_seconds: elapsed }
|
|
274
|
+
end
|
|
275
|
+
begin
|
|
276
|
+
socket.write("HTTP/1.1 408 Request Timeout\r\nconnection: close\r\ncontent-length: 0\r\n\r\n")
|
|
277
|
+
rescue StandardError
|
|
278
|
+
# Peer may have already gone — nothing to do.
|
|
279
|
+
end
|
|
280
|
+
@metrics.increment_status(408)
|
|
281
|
+
DEADLINE_SENTINEL
|
|
282
|
+
end
|
|
283
|
+
|
|
232
284
|
def chunked?(headers_part)
|
|
233
285
|
headers_part.match?(/^transfer-encoding:[ \t]*[^\r\n]*chunked\b/i)
|
|
234
286
|
end
|
|
@@ -36,33 +36,143 @@ module Hyperion
|
|
|
36
36
|
# Also exposes a `window_available` notification fan-out so the
|
|
37
37
|
# response-writer fiber can sleep until WINDOW_UPDATE arrives.
|
|
38
38
|
class RequestStream < ::Protocol::HTTP2::Stream
|
|
39
|
-
|
|
39
|
+
# RFC 7540 §8.1.2.1 — the only pseudo-headers a server MUST accept on a
|
|
40
|
+
# request. Anything else (notably `:status`, which is response-only, or
|
|
41
|
+
# an unknown `:foo`) is a malformed request that we reject with
|
|
42
|
+
# PROTOCOL_ERROR.
|
|
43
|
+
VALID_REQUEST_PSEUDO_HEADERS = %w[:method :path :scheme :authority].freeze
|
|
44
|
+
|
|
45
|
+
# RFC 7540 §8.1.2.2 — these connection-specific headers MUST NOT appear
|
|
46
|
+
# in HTTP/2 requests; their semantics are folded into HTTP/2 framing.
|
|
47
|
+
FORBIDDEN_HEADERS = %w[connection transfer-encoding keep-alive upgrade proxy-connection].freeze
|
|
48
|
+
|
|
49
|
+
attr_reader :request_headers, :request_body, :request_complete, :protocol_error_reason
|
|
40
50
|
|
|
41
51
|
def initialize(*)
|
|
42
52
|
super
|
|
43
53
|
@request_headers = []
|
|
44
54
|
@request_body = +''
|
|
55
|
+
@request_body_bytes = 0
|
|
45
56
|
@request_complete = false
|
|
46
57
|
@window_available = ::Async::Notification.new
|
|
58
|
+
@protocol_error_reason = nil
|
|
59
|
+
@declared_content_length = nil
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# Used by the dispatch loop to decide whether to invoke the app or
|
|
63
|
+
# send RST_STREAM PROTOCOL_ERROR. Set by `validate_request_headers!`
|
|
64
|
+
# and `validate_body_length!`.
|
|
65
|
+
def protocol_error?
|
|
66
|
+
!@protocol_error_reason.nil?
|
|
47
67
|
end
|
|
48
68
|
|
|
49
69
|
def process_headers(frame)
|
|
50
70
|
decoded = super
|
|
71
|
+
# First HEADERS frame on a stream carries the request header block;
|
|
72
|
+
# any later HEADERS frame is trailers (§8.1) and we deliberately do
|
|
73
|
+
# not re-validate (re-running the validator would see the original
|
|
74
|
+
# request pseudo-headers plus the new trailer block and falsely flag
|
|
75
|
+
# them as misordered).
|
|
76
|
+
first_block = @request_headers.empty?
|
|
51
77
|
# decoded is an Array of [name, value] pairs (HPACK output).
|
|
52
78
|
decoded.each { |pair| @request_headers << pair }
|
|
53
|
-
|
|
79
|
+
# Run RFC 7540 §8.1.2 validation as soon as we have a complete header
|
|
80
|
+
# block. We do it here (not at end_stream) so the dispatcher sees the
|
|
81
|
+
# error flag before it spawns a fiber for the request.
|
|
82
|
+
validate_request_headers! if first_block && !protocol_error?
|
|
83
|
+
if frame.end_stream?
|
|
84
|
+
validate_body_length! unless protocol_error?
|
|
85
|
+
@request_complete = true
|
|
86
|
+
end
|
|
54
87
|
decoded
|
|
55
88
|
end
|
|
56
89
|
|
|
57
90
|
def process_data(frame)
|
|
58
91
|
data = super
|
|
59
92
|
# rubocop:disable Rails/Present
|
|
60
|
-
|
|
93
|
+
if data && !data.empty?
|
|
94
|
+
@request_body << data
|
|
95
|
+
@request_body_bytes += data.bytesize
|
|
96
|
+
end
|
|
61
97
|
# rubocop:enable Rails/Present
|
|
62
|
-
|
|
98
|
+
if frame.end_stream?
|
|
99
|
+
validate_body_length! unless protocol_error?
|
|
100
|
+
@request_complete = true
|
|
101
|
+
end
|
|
63
102
|
data
|
|
64
103
|
end
|
|
65
104
|
|
|
105
|
+
# RFC 7540 §8.1.2 — request header validation. Sets
|
|
106
|
+
# `@protocol_error_reason` on the first violation we hit; the dispatch
|
|
107
|
+
# loop turns that into RST_STREAM PROTOCOL_ERROR.
|
|
108
|
+
def validate_request_headers!
|
|
109
|
+
seen_regular = false
|
|
110
|
+
pseudo_counts = Hash.new(0)
|
|
111
|
+
@request_headers.each do |pair|
|
|
112
|
+
name, value = pair
|
|
113
|
+
name = name.to_s
|
|
114
|
+
if name.start_with?(':')
|
|
115
|
+
# §8.1.2.1: pseudo-headers MUST precede regular headers.
|
|
116
|
+
return fail_validation!('pseudo-header after regular header') if seen_regular
|
|
117
|
+
# §8.1.2.1: only the four request pseudo-headers are valid; in
|
|
118
|
+
# particular, `:status` is response-only.
|
|
119
|
+
unless VALID_REQUEST_PSEUDO_HEADERS.include?(name)
|
|
120
|
+
return fail_validation!("invalid request pseudo-header: #{name}")
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
pseudo_counts[name] += 1
|
|
124
|
+
else
|
|
125
|
+
seen_regular = true
|
|
126
|
+
# §8.1.2: header names must be lowercase in HTTP/2.
|
|
127
|
+
return fail_validation!('uppercase header name') if /[A-Z]/.match?(name)
|
|
128
|
+
# §8.1.2.2: connection-specific headers are forbidden.
|
|
129
|
+
return fail_validation!("forbidden connection-specific header: #{name}") if FORBIDDEN_HEADERS.include?(name)
|
|
130
|
+
# §8.1.2.2: TE may only carry the value `trailers`.
|
|
131
|
+
if name == 'te' && value.to_s.downcase.strip != 'trailers'
|
|
132
|
+
return fail_validation!('TE header with non-trailers value')
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
# Track declared content-length for later body-byte cross-check.
|
|
136
|
+
@declared_content_length = value.to_s.to_i if name == 'content-length'
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
# §8.1.2.3: every pseudo-header may appear at most once.
|
|
141
|
+
pseudo_counts.each do |name, count|
|
|
142
|
+
return fail_validation!("duplicated pseudo-header: #{name}") if count > 1
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
method = pseudo_value(':method')
|
|
146
|
+
# CONNECT (§8.3) has its own rules; everything else MUST carry
|
|
147
|
+
# :method, :scheme and a non-empty :path.
|
|
148
|
+
if method == 'CONNECT'
|
|
149
|
+
return fail_validation!('CONNECT with :scheme') if pseudo_value(':scheme')
|
|
150
|
+
return fail_validation!('CONNECT with :path') if pseudo_value(':path')
|
|
151
|
+
return fail_validation!('CONNECT without :authority') unless pseudo_value(':authority')
|
|
152
|
+
else
|
|
153
|
+
return fail_validation!('missing :method') if method.nil? || method.empty?
|
|
154
|
+
|
|
155
|
+
scheme = pseudo_value(':scheme')
|
|
156
|
+
return fail_validation!('missing :scheme') if scheme.nil? || scheme.empty?
|
|
157
|
+
|
|
158
|
+
path = pseudo_value(':path')
|
|
159
|
+
return fail_validation!('missing or empty :path') if path.nil? || path.empty?
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
nil
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
# RFC 7540 §8.1.2.6 — if `content-length` was advertised, the actual
|
|
166
|
+
# number of DATA bytes received (across all DATA frames) MUST match.
|
|
167
|
+
def validate_body_length!
|
|
168
|
+
return if @declared_content_length.nil?
|
|
169
|
+
return if @declared_content_length == @request_body_bytes
|
|
170
|
+
|
|
171
|
+
fail_validation!(
|
|
172
|
+
"content-length mismatch: declared #{@declared_content_length}, received #{@request_body_bytes}"
|
|
173
|
+
)
|
|
174
|
+
end
|
|
175
|
+
|
|
66
176
|
# Called by protocol-http2 whenever the remote peer's flow-control
|
|
67
177
|
# window opens up — either via a stream-level WINDOW_UPDATE or via the
|
|
68
178
|
# connection-level fan-out in `Connection#consume_window`. We poke the
|
|
@@ -78,11 +188,58 @@ module Hyperion
|
|
|
78
188
|
def wait_for_window
|
|
79
189
|
@window_available.wait
|
|
80
190
|
end
|
|
191
|
+
|
|
192
|
+
private
|
|
193
|
+
|
|
194
|
+
# Look up a pseudo-header by name (e.g. `:method`) by scanning the raw
|
|
195
|
+
# collected pairs. Returns nil if absent. We don't pre-build a hash
|
|
196
|
+
# because the validator needs to detect duplicates first.
|
|
197
|
+
def pseudo_value(name)
|
|
198
|
+
@request_headers.each do |pair|
|
|
199
|
+
return pair[1].to_s if pair[0].to_s == name
|
|
200
|
+
end
|
|
201
|
+
nil
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
# Record the first protocol-error reason and short-circuit further
|
|
205
|
+
# validation. Returns nil so callers can `return fail_validation!(...)`.
|
|
206
|
+
def fail_validation!(reason)
|
|
207
|
+
@protocol_error_reason ||= reason
|
|
208
|
+
# As soon as a header-block violation is detected we treat the request
|
|
209
|
+
# as "complete" so the dispatch loop wakes up and emits RST_STREAM.
|
|
210
|
+
@request_complete = true
|
|
211
|
+
nil
|
|
212
|
+
end
|
|
81
213
|
end
|
|
82
214
|
|
|
83
|
-
|
|
215
|
+
# Maps Hyperion-friendly setting names to the integer SETTINGS_* identifiers
|
|
216
|
+
# protocol-http2 uses on the wire. See RFC 7540 §6.5.2 — these are the
|
|
217
|
+
# only four parameters Hyperion exposes; the rest of the SETTINGS frame
|
|
218
|
+
# (HEADER_TABLE_SIZE, ENABLE_PUSH, etc.) keeps protocol-http2's default.
|
|
219
|
+
SETTINGS_KEY_MAP = {
|
|
220
|
+
max_concurrent_streams: ::Protocol::HTTP2::Settings::MAXIMUM_CONCURRENT_STREAMS,
|
|
221
|
+
initial_window_size: ::Protocol::HTTP2::Settings::INITIAL_WINDOW_SIZE,
|
|
222
|
+
max_frame_size: ::Protocol::HTTP2::Settings::MAXIMUM_FRAME_SIZE,
|
|
223
|
+
max_header_list_size: ::Protocol::HTTP2::Settings::MAXIMUM_HEADER_LIST_SIZE
|
|
224
|
+
}.freeze
|
|
225
|
+
|
|
226
|
+
# RFC 7540 §6.5.2 floor for SETTINGS_MAX_FRAME_SIZE. protocol-http2 raises
|
|
227
|
+
# ProtocolError on values below this; we clamp + warn instead so a
|
|
228
|
+
# misconfigured operator gets a working server, not a boot-time crash.
|
|
229
|
+
H2_MIN_FRAME_SIZE = 0x4000 # 16384
|
|
230
|
+
|
|
231
|
+
# RFC 7540 §6.5.2 ceiling for SETTINGS_MAX_FRAME_SIZE.
|
|
232
|
+
H2_MAX_FRAME_SIZE = 0xFFFFFF # 16777215
|
|
233
|
+
|
|
234
|
+
# RFC 7540 §6.9.2 — INITIAL_WINDOW_SIZE has the same 31-bit max as the
|
|
235
|
+
# WINDOW_UPDATE frame's Window Size Increment (see protocol-http2's
|
|
236
|
+
# MAXIMUM_ALLOWED_WINDOW_SIZE).
|
|
237
|
+
H2_MAX_WINDOW_SIZE = 0x7FFFFFFF
|
|
238
|
+
|
|
239
|
+
def initialize(app:, thread_pool: nil, h2_settings: nil)
|
|
84
240
|
@app = app
|
|
85
241
|
@thread_pool = thread_pool
|
|
242
|
+
@h2_settings = h2_settings
|
|
86
243
|
@metrics = Hyperion.metrics
|
|
87
244
|
@logger = Hyperion.logger
|
|
88
245
|
end
|
|
@@ -92,7 +249,7 @@ module Hyperion
|
|
|
92
249
|
@metrics.increment(:connections_active)
|
|
93
250
|
framer = ::Protocol::HTTP2::Framer.new(socket)
|
|
94
251
|
server = build_server(framer)
|
|
95
|
-
server.read_connection_preface
|
|
252
|
+
server.read_connection_preface(initial_settings_payload)
|
|
96
253
|
|
|
97
254
|
# Extract once — the same TCP peer drives every stream on this conn.
|
|
98
255
|
peer_addr = peer_address(socket)
|
|
@@ -158,6 +315,69 @@ module Hyperion
|
|
|
158
315
|
|
|
159
316
|
private
|
|
160
317
|
|
|
318
|
+
# Build the [setting_id, value] pairs that go in the connection-preface
|
|
319
|
+
# SETTINGS frame. protocol-http2's Server#read_connection_preface accepts
|
|
320
|
+
# this array and does the wire encoding for us. Empty array (no overrides
|
|
321
|
+
# configured) → SETTINGS frame still goes out, just with no entries
|
|
322
|
+
# (effectively an ack), which is what the spec allows.
|
|
323
|
+
#
|
|
324
|
+
# We clamp out-of-range values (max_frame_size below the spec floor or
|
|
325
|
+
# above its ceiling, initial_window_size above 31-bit max) instead of
|
|
326
|
+
# letting protocol-http2 raise ProtocolError at handshake time — a
|
|
327
|
+
# crashing handshake leaks the connection. Operator gets a warn so the
|
|
328
|
+
# misconfiguration surfaces in logs.
|
|
329
|
+
def initial_settings_payload
|
|
330
|
+
return [] unless @h2_settings
|
|
331
|
+
|
|
332
|
+
payload = []
|
|
333
|
+
@h2_settings.each do |key, value|
|
|
334
|
+
next if value.nil?
|
|
335
|
+
|
|
336
|
+
setting_id = SETTINGS_KEY_MAP[key]
|
|
337
|
+
unless setting_id
|
|
338
|
+
@logger.warn { { message: 'unknown h2 setting; skipping', setting: key } }
|
|
339
|
+
next
|
|
340
|
+
end
|
|
341
|
+
|
|
342
|
+
clamped = clamp_h2_setting(key, value)
|
|
343
|
+
payload << [setting_id, clamped]
|
|
344
|
+
end
|
|
345
|
+
payload
|
|
346
|
+
end
|
|
347
|
+
|
|
348
|
+
def clamp_h2_setting(key, value)
|
|
349
|
+
case key
|
|
350
|
+
when :max_frame_size
|
|
351
|
+
if value < H2_MIN_FRAME_SIZE
|
|
352
|
+
@logger.warn do
|
|
353
|
+
{ message: 'h2 max_frame_size below spec minimum; clamping',
|
|
354
|
+
configured: value, clamped_to: H2_MIN_FRAME_SIZE }
|
|
355
|
+
end
|
|
356
|
+
H2_MIN_FRAME_SIZE
|
|
357
|
+
elsif value > H2_MAX_FRAME_SIZE
|
|
358
|
+
@logger.warn do
|
|
359
|
+
{ message: 'h2 max_frame_size above spec maximum; clamping',
|
|
360
|
+
configured: value, clamped_to: H2_MAX_FRAME_SIZE }
|
|
361
|
+
end
|
|
362
|
+
H2_MAX_FRAME_SIZE
|
|
363
|
+
else
|
|
364
|
+
value
|
|
365
|
+
end
|
|
366
|
+
when :initial_window_size
|
|
367
|
+
if value > H2_MAX_WINDOW_SIZE
|
|
368
|
+
@logger.warn do
|
|
369
|
+
{ message: 'h2 initial_window_size above spec maximum; clamping',
|
|
370
|
+
configured: value, clamped_to: H2_MAX_WINDOW_SIZE }
|
|
371
|
+
end
|
|
372
|
+
H2_MAX_WINDOW_SIZE
|
|
373
|
+
else
|
|
374
|
+
value
|
|
375
|
+
end
|
|
376
|
+
else
|
|
377
|
+
value
|
|
378
|
+
end
|
|
379
|
+
end
|
|
380
|
+
|
|
161
381
|
def build_server(framer)
|
|
162
382
|
server = ::Protocol::HTTP2::Server.new(framer)
|
|
163
383
|
server.define_singleton_method(:accept_stream) do |stream_id, &block|
|
|
@@ -175,6 +395,23 @@ module Hyperion
|
|
|
175
395
|
end
|
|
176
396
|
|
|
177
397
|
def dispatch_stream(stream, send_mutex, peer_addr = nil)
|
|
398
|
+
# RFC 7540 §8.1.2 — header validation flagged this stream as malformed.
|
|
399
|
+
# Send RST_STREAM PROTOCOL_ERROR instead of invoking the app.
|
|
400
|
+
if stream.protocol_error?
|
|
401
|
+
@logger.debug do
|
|
402
|
+
{ message: 'h2 request rejected', reason: stream.protocol_error_reason, stream_id: stream.id }
|
|
403
|
+
end
|
|
404
|
+
@metrics.increment(:requests_rejected)
|
|
405
|
+
begin
|
|
406
|
+
send_mutex.synchronize do
|
|
407
|
+
stream.send_reset_stream(::Protocol::HTTP2::Error::PROTOCOL_ERROR) unless stream.closed?
|
|
408
|
+
end
|
|
409
|
+
rescue StandardError
|
|
410
|
+
nil
|
|
411
|
+
end
|
|
412
|
+
return
|
|
413
|
+
end
|
|
414
|
+
|
|
178
415
|
pseudo, regular = partition_pseudo(stream.request_headers)
|
|
179
416
|
|
|
180
417
|
method = pseudo[':method'] || 'GET'
|
data/lib/hyperion/logger.rb
CHANGED
|
@@ -64,6 +64,34 @@ module Hyperion
|
|
|
64
64
|
# Colorize when format is text AND the destination is a TTY. We only
|
|
65
65
|
# check the regular stream here — colored text is for humans.
|
|
66
66
|
@colorize = @format == :text && tty?(@out)
|
|
67
|
+
@c_access_available = nil # lazy-computed on first access — see below.
|
|
68
|
+
# Registry of every per-thread access buffer ever allocated through
|
|
69
|
+
# this Logger instance. Walked by #flush_all on shutdown so SIGTERM
|
|
70
|
+
# doesn't strand buffered lines in dying threads. The Mutex guards
|
|
71
|
+
# registration on first allocation per thread (rare) and the shutdown
|
|
72
|
+
# walk; the hot #access path stays lock-free.
|
|
73
|
+
@access_buffers = []
|
|
74
|
+
@access_buffers_mutex = Mutex.new
|
|
75
|
+
# Per-instance thread-local key. A globally-shared key (e.g. a frozen
|
|
76
|
+
# Symbol constant) lets a buffer created by an earlier Logger in this
|
|
77
|
+
# thread be picked up by a later Logger — but the buffer is registered
|
|
78
|
+
# against the *earlier* Logger's @access_buffers, so the new Logger's
|
|
79
|
+
# #flush_all can't see it. Namespacing the key per-instance fixes that:
|
|
80
|
+
# each Logger gets its own per-thread buffer, and the registry it
|
|
81
|
+
# walks at shutdown matches the one #access wrote to. The Symbol is
|
|
82
|
+
# allocated once at construction; the hot path just reads it.
|
|
83
|
+
@buffer_key = :"__hyperion_access_buf_#{object_id}__"
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# Whether Hyperion::CParser.build_access_line is available. Probed lazily
|
|
87
|
+
# on first call (the C parser is required after Logger is required, so we
|
|
88
|
+
# can't cache this at constant-define time — it would always be false).
|
|
89
|
+
# Memoised per-instance to keep the hot path branchless.
|
|
90
|
+
def c_access_available?
|
|
91
|
+
return @c_access_available unless @c_access_available.nil?
|
|
92
|
+
|
|
93
|
+
@c_access_available = defined?(::Hyperion::CParser) &&
|
|
94
|
+
::Hyperion::CParser.respond_to?(:build_access_line)
|
|
67
95
|
end
|
|
68
96
|
|
|
69
97
|
LEVELS.each_key do |lvl|
|
|
@@ -106,13 +134,23 @@ module Hyperion
|
|
|
106
134
|
return unless emit?(:info)
|
|
107
135
|
|
|
108
136
|
ts = cached_timestamp
|
|
109
|
-
line
|
|
137
|
+
# The C extension builds the line in a stack scratch buffer (~10× faster
|
|
138
|
+
# than the Ruby interpolation path). It only fires when colorization is
|
|
139
|
+
# off — a colored TTY line needs ANSI escapes around the level label,
|
|
140
|
+
# which the C builder doesn't emit. Production deploys (non-TTY,
|
|
141
|
+
# log-aggregator destinations) take the C path; local TTY runs keep the
|
|
142
|
+
# colored Ruby fallback.
|
|
143
|
+
line = if !@colorize && c_access_available?
|
|
144
|
+
::Hyperion::CParser.build_access_line(@format, ts, method, path,
|
|
145
|
+
query, status, duration_ms,
|
|
146
|
+
remote_addr, http_version)
|
|
147
|
+
elsif @format == :json
|
|
110
148
|
build_access_json(ts, method, path, query, status, duration_ms, remote_addr, http_version)
|
|
111
149
|
else
|
|
112
150
|
build_access_text(ts, method, path, query, status, duration_ms, remote_addr, http_version)
|
|
113
151
|
end
|
|
114
152
|
|
|
115
|
-
buf =
|
|
153
|
+
buf = Thread.current[@buffer_key] || allocate_access_buffer
|
|
116
154
|
buf << line
|
|
117
155
|
return if buf.bytesize < ACCESS_FLUSH_BYTES
|
|
118
156
|
|
|
@@ -126,7 +164,7 @@ module Hyperion
|
|
|
126
164
|
# loop when a connection closes (so log lines from a closing keep-alive
|
|
127
165
|
# session don't get stuck behind the buffer until the next connection).
|
|
128
166
|
def flush_access_buffer
|
|
129
|
-
buf = Thread.current[
|
|
167
|
+
buf = Thread.current[@buffer_key]
|
|
130
168
|
return if buf.nil? || buf.empty?
|
|
131
169
|
|
|
132
170
|
@out.write(buf)
|
|
@@ -135,8 +173,61 @@ module Hyperion
|
|
|
135
173
|
# Swallow logger failures — never let logging crash the server.
|
|
136
174
|
end
|
|
137
175
|
|
|
176
|
+
# Flush every per-thread access-log buffer ever allocated through this
|
|
177
|
+
# Logger, then sync the underlying IOs.
|
|
178
|
+
#
|
|
179
|
+
# Why this exists: under SIGTERM, Master#shutdown_children logs the
|
|
180
|
+
# 'master draining' / 'master exiting' lines and then exits. The 'info'
|
|
181
|
+
# path doesn't go through the access buffer, but it does rely on glibc
|
|
182
|
+
# stdio buffering being flushed before the process dies — and per-thread
|
|
183
|
+
# access buffers (Thread.current[:__hyperion_access_buf__]) are *only*
|
|
184
|
+
# flushed when the buffer reaches ACCESS_FLUSH_BYTES or when the owning
|
|
185
|
+
# thread closes a connection. On a clean SIGTERM both can be missed and
|
|
186
|
+
# the operator sees nothing in the captured log. This method walks every
|
|
187
|
+
# registered per-thread buffer, writes any pending bytes, then calls
|
|
188
|
+
# IO#flush on @out / @err so the kernel sees them before exec_exit.
|
|
189
|
+
#
|
|
190
|
+
# Safe to call from any thread. Idempotent. Never raises.
|
|
191
|
+
def flush_all
|
|
192
|
+
buffers = @access_buffers_mutex.synchronize { @access_buffers.dup }
|
|
193
|
+
buffers.each do |buf|
|
|
194
|
+
next if buf.empty?
|
|
195
|
+
|
|
196
|
+
begin
|
|
197
|
+
@out.write(buf)
|
|
198
|
+
buf.clear
|
|
199
|
+
rescue StandardError
|
|
200
|
+
# Continue — one bad buffer must not block the rest.
|
|
201
|
+
end
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
flush_io(@out)
|
|
205
|
+
flush_io(@err) unless @err.equal?(@out)
|
|
206
|
+
rescue StandardError
|
|
207
|
+
# Swallow logger failures — never let logging crash the server.
|
|
208
|
+
end
|
|
209
|
+
|
|
138
210
|
private
|
|
139
211
|
|
|
212
|
+
# First-touch path for a thread's access buffer. Allocates the String,
|
|
213
|
+
# stores it in the thread-local for lock-free access on subsequent calls,
|
|
214
|
+
# and registers it in @access_buffers so #flush_all can find it later.
|
|
215
|
+
# Mutex is taken once per thread (not per request).
|
|
216
|
+
def allocate_access_buffer
|
|
217
|
+
buf = +''
|
|
218
|
+
Thread.current[@buffer_key] = buf
|
|
219
|
+
@access_buffers_mutex.synchronize { @access_buffers << buf }
|
|
220
|
+
buf
|
|
221
|
+
end
|
|
222
|
+
|
|
223
|
+
def flush_io(io)
|
|
224
|
+
io.flush if io.respond_to?(:flush)
|
|
225
|
+
rescue StandardError
|
|
226
|
+
# Some IO destinations raise on flush (closed pipes during SIGPIPE,
|
|
227
|
+
# custom IO-likes that don't implement it cleanly). Logging must
|
|
228
|
+
# never crash the server, especially during shutdown.
|
|
229
|
+
end
|
|
230
|
+
|
|
140
231
|
# Cached UTC iso8601(3) timestamp, refreshed at most once per millisecond
|
|
141
232
|
# per thread. At 24k r/s with 16 threads we render ~1500 r/s/thread; only
|
|
142
233
|
# ~1000 of those allocate a new String. The other 500 reuse the cached one.
|
data/lib/hyperion/master.rb
CHANGED
|
@@ -47,6 +47,20 @@ module Hyperion
|
|
|
47
47
|
end
|
|
48
48
|
end
|
|
49
49
|
|
|
50
|
+
# Pulls the four configurable HTTP/2 SETTINGS values out of the Config
|
|
51
|
+
# and returns them as a Hash. Nils are stripped so an operator who
|
|
52
|
+
# explicitly sets one to `nil` (meaning "leave protocol-http2 default in
|
|
53
|
+
# place") doesn't accidentally send a SETTINGS entry with a nil value.
|
|
54
|
+
# Empty hash → no override → Http2Handler skips the SETTINGS push.
|
|
55
|
+
def self.build_h2_settings(config)
|
|
56
|
+
{
|
|
57
|
+
max_concurrent_streams: config.h2_max_concurrent_streams,
|
|
58
|
+
initial_window_size: config.h2_initial_window_size,
|
|
59
|
+
max_frame_size: config.h2_max_frame_size,
|
|
60
|
+
max_header_list_size: config.h2_max_header_list_size
|
|
61
|
+
}.compact
|
|
62
|
+
end
|
|
63
|
+
|
|
50
64
|
def initialize(host:, port:, app:, workers: DEFAULT_WORKER_COUNT,
|
|
51
65
|
read_timeout: Server::DEFAULT_READ_TIMEOUT_SECONDS, tls: nil,
|
|
52
66
|
thread_count: Server::DEFAULT_THREAD_COUNT, config: nil)
|
|
@@ -64,6 +78,10 @@ module Hyperion
|
|
|
64
78
|
@stopping = false
|
|
65
79
|
@worker_model = self.class.detect_worker_model
|
|
66
80
|
@listener = nil # populated only in :share mode
|
|
81
|
+
@worker_max_rss_mb = @config.worker_max_rss_mb
|
|
82
|
+
@worker_check_interval = @config.worker_check_interval || 30
|
|
83
|
+
@last_health_check = 0 # monotonic seconds
|
|
84
|
+
@cycling = {} # pid => true while we wait for it to exit
|
|
67
85
|
end
|
|
68
86
|
|
|
69
87
|
def run
|
|
@@ -80,6 +98,12 @@ module Hyperion
|
|
|
80
98
|
}
|
|
81
99
|
end
|
|
82
100
|
|
|
101
|
+
# Pre-allocate Rack env-pool entries and eager-touch lazy constants
|
|
102
|
+
# BEFORE we fork. Children inherit the warm memory via copy-on-write
|
|
103
|
+
# so the first batch of requests on each fresh worker doesn't pay
|
|
104
|
+
# the allocation/autoload tax.
|
|
105
|
+
Hyperion.warmup!
|
|
106
|
+
|
|
83
107
|
# `before_fork` runs ONCE in the master before any worker is forked.
|
|
84
108
|
# Operators use it to close shared resources (DB pools, Redis sockets)
|
|
85
109
|
# so each child gets fresh connections rather than inheriting the
|
|
@@ -139,7 +163,10 @@ module Hyperion
|
|
|
139
163
|
host: @host, port: @port, app: @app,
|
|
140
164
|
read_timeout: @read_timeout, tls: @tls,
|
|
141
165
|
thread_count: @thread_count, config: @config,
|
|
142
|
-
worker_index: worker_index
|
|
166
|
+
worker_index: worker_index,
|
|
167
|
+
max_pending: @config.max_pending,
|
|
168
|
+
max_request_read_seconds: @config.max_request_read_seconds,
|
|
169
|
+
h2_settings: Master.build_h2_settings(@config)
|
|
143
170
|
}
|
|
144
171
|
# Hand the inherited socket to the worker in :share mode. In
|
|
145
172
|
# :reuseport mode the worker binds its own with SO_REUSEPORT.
|
|
@@ -165,6 +192,7 @@ module Hyperion
|
|
|
165
192
|
end
|
|
166
193
|
|
|
167
194
|
reap_and_respawn
|
|
195
|
+
maybe_cycle_workers
|
|
168
196
|
end
|
|
169
197
|
|
|
170
198
|
shutdown_children
|
|
@@ -177,12 +205,47 @@ module Hyperion
|
|
|
177
205
|
|
|
178
206
|
Hyperion.logger.warn { { message: 'worker died, respawning', worker_pid: pid } }
|
|
179
207
|
@children.delete(pid)
|
|
208
|
+
@cycling.delete(pid)
|
|
180
209
|
spawn_worker unless @stopping
|
|
181
210
|
end
|
|
182
211
|
rescue Errno::ECHILD
|
|
183
212
|
# No children — happens during shutdown.
|
|
184
213
|
end
|
|
185
214
|
|
|
215
|
+
# Periodically poll worker RSS and SIGTERM any that exceed the configured
|
|
216
|
+
# cap. The dying worker is reaped by `reap_and_respawn` on the next tick,
|
|
217
|
+
# which also clears the @cycling guard so the slot can be replaced.
|
|
218
|
+
# Skips entirely when no cap is configured — zero overhead by default.
|
|
219
|
+
def maybe_cycle_workers
|
|
220
|
+
return unless @worker_max_rss_mb
|
|
221
|
+
|
|
222
|
+
now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
223
|
+
return if now - @last_health_check < @worker_check_interval
|
|
224
|
+
|
|
225
|
+
@last_health_check = now
|
|
226
|
+
@children.each_key do |pid|
|
|
227
|
+
next if @cycling.key?(pid)
|
|
228
|
+
|
|
229
|
+
rss = WorkerHealth.rss_mb(pid)
|
|
230
|
+
next unless rss && rss > @worker_max_rss_mb
|
|
231
|
+
|
|
232
|
+
Hyperion.logger.warn do
|
|
233
|
+
{
|
|
234
|
+
message: 'cycling worker for memory',
|
|
235
|
+
worker_pid: pid,
|
|
236
|
+
rss_mb: rss,
|
|
237
|
+
limit_mb: @worker_max_rss_mb
|
|
238
|
+
}
|
|
239
|
+
end
|
|
240
|
+
@cycling[pid] = true
|
|
241
|
+
begin
|
|
242
|
+
Process.kill('TERM', pid)
|
|
243
|
+
rescue StandardError
|
|
244
|
+
# process already gone — reap_and_respawn will handle it
|
|
245
|
+
end
|
|
246
|
+
end
|
|
247
|
+
end
|
|
248
|
+
|
|
186
249
|
def shutdown_children
|
|
187
250
|
Hyperion.logger.info do
|
|
188
251
|
{ message: 'master draining', graceful_timeout: @graceful_timeout }
|
|
@@ -216,6 +279,11 @@ module Hyperion
|
|
|
216
279
|
@children.clear
|
|
217
280
|
|
|
218
281
|
Hyperion.logger.info { { message: 'master exiting' } }
|
|
282
|
+
# Drain per-thread access buffers + sync stdio so the 'master draining'
|
|
283
|
+
# / 'master exiting' lines (and any in-flight access-log lines from
|
|
284
|
+
# threads that never reached the 4-KiB flush threshold) actually reach
|
|
285
|
+
# the operator's log file before the process exits on SIGTERM.
|
|
286
|
+
Hyperion.logger.flush_all
|
|
219
287
|
end
|
|
220
288
|
end
|
|
221
289
|
end
|