tipi 0.41 → 0.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/test.yml +4 -0
- data/.gitignore +3 -1
- data/CHANGELOG.md +11 -0
- data/Gemfile +5 -1
- data/Gemfile.lock +26 -12
- data/benchmarks/bm_http1_parser.rb +61 -0
- data/bin/benchmark +37 -0
- data/bin/h1pd +6 -0
- data/bin/tipi +3 -21
- data/df/server.rb +1 -1
- data/df/server_utils.rb +49 -44
- data/examples/full_service.rb +13 -0
- data/examples/http1_parser.rb +10 -8
- data/examples/http_server.rb +4 -1
- data/examples/https_server.rb +3 -0
- data/examples/servername_cb.rb +37 -0
- data/ext/tipi/extconf.rb +3 -2
- data/ext/tipi/http1_parser.c +478 -189
- data/lib/tipi.rb +84 -3
- data/lib/tipi/acme.rb +308 -0
- data/lib/tipi/cli.rb +30 -0
- data/lib/tipi/digital_fabric/agent.rb +2 -2
- data/lib/tipi/digital_fabric/agent_proxy.rb +4 -3
- data/lib/tipi/digital_fabric/executive.rb +6 -2
- data/lib/tipi/digital_fabric/protocol.rb +2 -2
- data/lib/tipi/digital_fabric/request_adapter.rb +0 -4
- data/lib/tipi/digital_fabric/service.rb +5 -10
- data/lib/tipi/http1_adapter.rb +55 -100
- data/lib/tipi/http2_adapter.rb +19 -6
- data/lib/tipi/http2_stream.rb +39 -43
- data/lib/tipi/version.rb +1 -1
- data/security/http1.rb +12 -0
- data/test/helper.rb +60 -11
- data/test/test_http1_parser.rb +586 -0
- data/test/test_http_server.rb +0 -27
- data/test/test_request.rb +1 -28
- data/tipi.gemspec +6 -5
- metadata +48 -27
- data/examples/automatic_certificate.rb +0 -193
- data/lib/tipi/http1_adapter_new.rb +0 -293
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 82155f4b86223d3fb32dc8c314abfeb75fda951747e63832ed5c15bc3b1f43cc
|
4
|
+
data.tar.gz: 721b98fb8d330d1bc38f8f236f82ddd909235d31f5875c1b4f89895e4e3926da
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 10389779975234d90c968626b4f9d6b168491f9a32ecf1be71683c37cf11f58f1bac52a36478691e72328236a16d309a8462104bda556772e25cebe52b2b0d53
|
7
|
+
data.tar.gz: 99db2c15278cdfc286ffad05f266f5d62eec6a3bb1363f8dfa4c302549a308a691496ee60ec085ede59e3b594533e8e196bbca144878e5a6dccf2ecfc962151c
|
data/.github/workflows/test.yml
CHANGED
data/.gitignore
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,14 @@
|
|
1
|
+
## 0.42 2021-08-16
|
2
|
+
|
3
|
+
- HTTP/1 parser: disable UTF-8 parsing for all but header values
|
4
|
+
- Add support for parsing HTTP/1 from callable source
|
5
|
+
- Introduce full_service API for automatic HTTPS
|
6
|
+
- Introduce automatic SSL certificate provisioning
|
7
|
+
- Improve handling of exceptions
|
8
|
+
- Various fixes to DF service and agent pxoy
|
9
|
+
- Fix upgrading to HTTP2 with a request body
|
10
|
+
- Switch to new HTTP/1 parser
|
11
|
+
|
1
12
|
## 0.41 2021-07-26
|
2
13
|
|
3
14
|
- Fix Rack adapter (#11)
|
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,13 +1,25 @@
|
|
1
|
+
PATH
|
2
|
+
remote: ../polyphony
|
3
|
+
specs:
|
4
|
+
polyphony (0.69)
|
5
|
+
|
6
|
+
PATH
|
7
|
+
remote: ../qeweney
|
8
|
+
specs:
|
9
|
+
qeweney (0.14)
|
10
|
+
escape_utils (~> 1.2.1)
|
11
|
+
|
1
12
|
PATH
|
2
13
|
remote: .
|
3
14
|
specs:
|
4
|
-
tipi (0.
|
15
|
+
tipi (0.42)
|
5
16
|
acme-client (~> 2.0.8)
|
6
|
-
|
7
|
-
|
17
|
+
extralite (~> 1.2)
|
18
|
+
http-2 (~> 0.11)
|
19
|
+
localhost (~> 1.1.4)
|
8
20
|
msgpack (~> 1.4.2)
|
9
|
-
polyphony (~> 0.
|
10
|
-
qeweney (~> 0.
|
21
|
+
polyphony (~> 0.69)
|
22
|
+
qeweney (~> 0.14)
|
11
23
|
rack (>= 2.0.8, < 2.3.0)
|
12
24
|
websocket (~> 1.2.8)
|
13
25
|
|
@@ -22,7 +34,8 @@ GEM
|
|
22
34
|
rack (>= 1.6.0)
|
23
35
|
docile (1.4.0)
|
24
36
|
escape_utils (1.2.1)
|
25
|
-
|
37
|
+
extralite (1.2)
|
38
|
+
faraday (1.7.0)
|
26
39
|
faraday-em_http (~> 1.0)
|
27
40
|
faraday-em_synchrony (~> 1.0)
|
28
41
|
faraday-excon (~> 1.1)
|
@@ -30,6 +43,7 @@ GEM
|
|
30
43
|
faraday-net_http (~> 1.0)
|
31
44
|
faraday-net_http_persistent (~> 1.1)
|
32
45
|
faraday-patron (~> 1.0)
|
46
|
+
faraday-rack (~> 1.0)
|
33
47
|
multipart-post (>= 1.2, < 3)
|
34
48
|
ruby2_keywords (>= 0.0.4)
|
35
49
|
faraday-em_http (1.0.0)
|
@@ -39,8 +53,9 @@ GEM
|
|
39
53
|
faraday-net_http (1.0.1)
|
40
54
|
faraday-net_http_persistent (1.2.0)
|
41
55
|
faraday-patron (1.0.0)
|
42
|
-
|
43
|
-
|
56
|
+
faraday-rack (1.0.0)
|
57
|
+
http-2 (0.11.0)
|
58
|
+
http_parser.rb (0.7.0)
|
44
59
|
json (2.5.1)
|
45
60
|
localhost (1.1.8)
|
46
61
|
minitest (5.11.3)
|
@@ -51,9 +66,6 @@ GEM
|
|
51
66
|
ruby-progressbar
|
52
67
|
msgpack (1.4.2)
|
53
68
|
multipart-post (2.1.1)
|
54
|
-
polyphony (0.64)
|
55
|
-
qeweney (0.11)
|
56
|
-
escape_utils (~> 1.2.1)
|
57
69
|
rack (2.2.3)
|
58
70
|
rake (12.3.3)
|
59
71
|
rake-compiler (1.1.1)
|
@@ -72,9 +84,11 @@ PLATFORMS
|
|
72
84
|
|
73
85
|
DEPENDENCIES
|
74
86
|
cuba (~> 3.9.3)
|
75
|
-
|
87
|
+
http_parser.rb (= 0.7.0)
|
76
88
|
minitest (~> 5.11.3)
|
77
89
|
minitest-reporters (~> 1.4.2)
|
90
|
+
polyphony!
|
91
|
+
qeweney!
|
78
92
|
rake (~> 12.3.3)
|
79
93
|
rake-compiler (= 1.1.1)
|
80
94
|
simplecov (~> 0.17.1)
|
@@ -0,0 +1,61 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'bundler/setup'
|
4
|
+
|
5
|
+
HTTP_REQUEST = "GET /foo HTTP/1.1\r\nHost: example.com\r\nAccept: */*\r\n\r\n"
|
6
|
+
|
7
|
+
def benchmark_other_http1_parser(iterations)
|
8
|
+
STDOUT << "http_parser.rb: "
|
9
|
+
require 'http_parser.rb'
|
10
|
+
|
11
|
+
i, o = IO.pipe
|
12
|
+
parser = Http::Parser.new
|
13
|
+
done = false
|
14
|
+
headers = nil
|
15
|
+
parser.on_headers_complete = proc do |h|
|
16
|
+
headers = h
|
17
|
+
headers[':method'] = parser.http_method
|
18
|
+
headers[':path'] = parser.request_url
|
19
|
+
headers[':protocol'] = parser.http_version
|
20
|
+
end
|
21
|
+
parser.on_message_complete = proc { done = true }
|
22
|
+
|
23
|
+
t0 = Time.now
|
24
|
+
iterations.times do
|
25
|
+
o << HTTP_REQUEST
|
26
|
+
done = false
|
27
|
+
while !done
|
28
|
+
msg = i.readpartial(4096)
|
29
|
+
parser << msg
|
30
|
+
end
|
31
|
+
end
|
32
|
+
t1 = Time.now
|
33
|
+
puts "#{iterations / (t1 - t0)} ips"
|
34
|
+
end
|
35
|
+
|
36
|
+
def benchmark_tipi_http1_parser(iterations)
|
37
|
+
STDOUT << "tipi parser: "
|
38
|
+
require_relative '../lib/tipi_ext'
|
39
|
+
i, o = IO.pipe
|
40
|
+
reader = proc { |len| i.readpartial(len) }
|
41
|
+
parser = Tipi::HTTP1Parser.new(reader)
|
42
|
+
|
43
|
+
t0 = Time.now
|
44
|
+
iterations.times do
|
45
|
+
o << HTTP_REQUEST
|
46
|
+
headers = parser.parse_headers
|
47
|
+
end
|
48
|
+
t1 = Time.now
|
49
|
+
puts "#{iterations / (t1 - t0)} ips"
|
50
|
+
end
|
51
|
+
|
52
|
+
def fork_benchmark(method, iterations)
|
53
|
+
pid = fork { send(method, iterations) }
|
54
|
+
Process.wait(pid)
|
55
|
+
end
|
56
|
+
|
57
|
+
x = 500000
|
58
|
+
fork_benchmark(:benchmark_other_http1_parser, x)
|
59
|
+
fork_benchmark(:benchmark_tipi_http1_parser, x)
|
60
|
+
|
61
|
+
# benchmark_tipi_http1_parser(x)
|
data/bin/benchmark
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'bundler/setup'
|
4
|
+
require 'polyphony'
|
5
|
+
|
6
|
+
def parse_latency(latency)
|
7
|
+
m = latency.match(/^([\d\.]+)(us|ms|s)$/)
|
8
|
+
return nil unless m
|
9
|
+
|
10
|
+
value = m[1].to_f
|
11
|
+
case m[2]
|
12
|
+
when 's' then value
|
13
|
+
when 'ms' then value / 1000
|
14
|
+
when 'us' then value / 1000000
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def parse_wrk_results(results)
|
19
|
+
lines = results.lines
|
20
|
+
latencies = lines[3].strip.split(/\s+/)
|
21
|
+
throughput = lines[6].strip.split(/\s+/)
|
22
|
+
|
23
|
+
{
|
24
|
+
latency_avg: parse_latency(latencies[1]),
|
25
|
+
latency_max: parse_latency(latencies[3]),
|
26
|
+
rate: throughput[1].to_f
|
27
|
+
}
|
28
|
+
end
|
29
|
+
|
30
|
+
def run_wrk(duration: 10, threads: 2, connections: 10, url: )
|
31
|
+
`wrk -d#{duration} -t#{threads} -c#{connections} #{url}`
|
32
|
+
end
|
33
|
+
|
34
|
+
[8, 64, 256, 512].each do |c|
|
35
|
+
puts "connections: #{c}"
|
36
|
+
p parse_wrk_results(run_wrk(duration: 10, threads: 4, connections: c, url: "http://localhost:10080/"))
|
37
|
+
end
|
data/bin/h1pd
ADDED
data/bin/tipi
CHANGED
@@ -1,26 +1,8 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
|
3
3
|
require 'bundler/setup'
|
4
|
-
require '
|
5
|
-
require File.expand_path('../lib/tipi/configuration', __dir__)
|
4
|
+
require 'tipi/cli'
|
6
5
|
|
7
|
-
|
8
|
-
#config[:forked] = 4
|
6
|
+
trap('SIGINT') { exit }
|
9
7
|
|
10
|
-
|
11
|
-
puts
|
12
|
-
|
13
|
-
configuration_manager = spin { Tipi::Configuration.supervise_config }
|
14
|
-
|
15
|
-
configuration_manager << config
|
16
|
-
configuration_manager.await
|
17
|
-
|
18
|
-
__END__
|
19
|
-
|
20
|
-
ooo
|
21
|
-
oo
|
22
|
-
o
|
23
|
-
\|/
|
24
|
-
/ \ Tipi - A better web server for a better world
|
25
|
-
/___\
|
26
|
-
|
8
|
+
Tipi::CLI.start
|
data/df/server.rb
CHANGED
@@ -17,7 +17,7 @@ rescue Interrupt
|
|
17
17
|
rescue SystemExit
|
18
18
|
# ignore
|
19
19
|
rescue Exception => e
|
20
|
-
log("Uncaught exception", error: e, backtrace: e.backtrace)
|
20
|
+
log("Uncaught exception", error: e, source: e.source_fiber, raising: e.raising_fiber, backtrace: e.backtrace)
|
21
21
|
ensure
|
22
22
|
log('DF server stopped')
|
23
23
|
end
|
data/df/server_utils.rb
CHANGED
@@ -7,24 +7,7 @@ require 'tipi/digital_fabric/executive'
|
|
7
7
|
require 'json'
|
8
8
|
require 'fileutils'
|
9
9
|
require 'time'
|
10
|
-
|
11
|
-
module ::Kernel
|
12
|
-
def trace(*args)
|
13
|
-
STDOUT.orig_write(format_trace(args))
|
14
|
-
end
|
15
|
-
|
16
|
-
def format_trace(args)
|
17
|
-
if args.first.is_a?(String)
|
18
|
-
if args.size > 1
|
19
|
-
format("%s: %p\n", args.shift, args)
|
20
|
-
else
|
21
|
-
format("%s\n", args.first)
|
22
|
-
end
|
23
|
-
else
|
24
|
-
format("%p\n", args.size == 1 ? args.first : args)
|
25
|
-
end
|
26
|
-
end
|
27
|
-
end
|
10
|
+
require 'polyphony/extensions/debug'
|
28
11
|
|
29
12
|
FileUtils.cd(__dir__)
|
30
13
|
|
@@ -65,8 +48,10 @@ def listen_http
|
|
65
48
|
# log("Done with HTTP connection", client: client)
|
66
49
|
@service.decr_connection_count
|
67
50
|
end
|
68
|
-
rescue
|
69
|
-
|
51
|
+
rescue Polyphony::BaseException
|
52
|
+
raise
|
53
|
+
rescue Exception => e
|
54
|
+
log 'HTTP accept (unknown) error', error: e, backtrace: e.backtrace
|
70
55
|
end
|
71
56
|
end
|
72
57
|
end
|
@@ -74,7 +59,7 @@ end
|
|
74
59
|
CERTIFICATE_REGEXP = /(-----BEGIN CERTIFICATE-----\n[^-]+-----END CERTIFICATE-----\n)/.freeze
|
75
60
|
|
76
61
|
def listen_https
|
77
|
-
spin(
|
62
|
+
spin(:https_listener) do
|
78
63
|
private_key = OpenSSL::PKey::RSA.new IO.read('../../reality/ssl/privkey.pem')
|
79
64
|
c = IO.read('../../reality/ssl/cacert.pem')
|
80
65
|
certificates = c.scan(CERTIFICATE_REGEXP).map { |p| OpenSSL::X509::Certificate.new(p.first) }
|
@@ -83,6 +68,13 @@ def listen_https
|
|
83
68
|
log "SSL Certificate expires: #{cert.not_after.inspect}"
|
84
69
|
ctx.add_certificate(cert, private_key, certificates)
|
85
70
|
ctx.ciphers = 'ECDH+aRSA'
|
71
|
+
ctx.send(
|
72
|
+
:set_minmax_proto_version,
|
73
|
+
OpenSSL::SSL::SSL3_VERSION,
|
74
|
+
OpenSSL::SSL::TLS1_3_VERSION
|
75
|
+
)
|
76
|
+
# ctx.min_version = OpenSSL::SSL::SSL3_VERSION #OpenSSL::SSL::TLS1_VERSION
|
77
|
+
# ctx.max_version = OpenSSL::SSL::TLS1_3_VERSION
|
86
78
|
|
87
79
|
# TODO: further limit ciphers
|
88
80
|
# ref: https://github.com/socketry/falcon/blob/3ec805b3ceda0a764a2c5eb68cde33897b6a35ff/lib/falcon/environments/tls.rb
|
@@ -99,8 +91,10 @@ def listen_https
|
|
99
91
|
server = Polyphony::Net.tcp_listen('0.0.0.0', 10443, opts)
|
100
92
|
id = 0
|
101
93
|
loop do
|
94
|
+
log('Before HTTPS server.accept')
|
102
95
|
client = server.accept
|
103
|
-
|
96
|
+
log('After HTTPS server.accept')
|
97
|
+
log('Accept HTTPS client connection', client: client)
|
104
98
|
spin("https#{id += 1}") do
|
105
99
|
@service.incr_connection_count
|
106
100
|
Tipi.client_loop(client, opts) { |req| @service.http_request(req) }
|
@@ -111,9 +105,11 @@ def listen_https
|
|
111
105
|
@service.decr_connection_count
|
112
106
|
end
|
113
107
|
rescue OpenSSL::SSL::SSLError, SystemCallError, TypeError => e
|
114
|
-
|
115
|
-
rescue
|
116
|
-
|
108
|
+
log('HTTPS accept error', error: e)
|
109
|
+
rescue Polyphony::BaseException
|
110
|
+
raise
|
111
|
+
rescue Exception => e
|
112
|
+
log 'HTTPS accept (unknown) error', error: e, backtrace: e.backtrace
|
117
113
|
end
|
118
114
|
end
|
119
115
|
end
|
@@ -126,13 +122,16 @@ def listen_unix
|
|
126
122
|
socket = UNIXServer.new(UNIX_SOCKET_PATH)
|
127
123
|
|
128
124
|
id = 0
|
129
|
-
|
125
|
+
loop do
|
126
|
+
client = socket.accept
|
130
127
|
# log('Accept Unix connection', client: client)
|
131
128
|
spin("unix#{id += 1}") do
|
132
129
|
Tipi.client_loop(client, {}) { |req| @service.http_request(req, true) }
|
133
130
|
end
|
134
|
-
rescue
|
135
|
-
|
131
|
+
rescue Polyphony::BaseException
|
132
|
+
raise
|
133
|
+
rescue Exception => e
|
134
|
+
log 'Unix accept error', error: e, backtrace: e.backtrace
|
136
135
|
end
|
137
136
|
end
|
138
137
|
end
|
@@ -141,33 +140,39 @@ def listen_df
|
|
141
140
|
spin(:df_listener) do
|
142
141
|
opts = {
|
143
142
|
reuse_addr: true,
|
143
|
+
reuse_port: true,
|
144
144
|
dont_linger: true,
|
145
145
|
}
|
146
146
|
log('Listening for DF connections on localhost:4321')
|
147
147
|
server = Polyphony::Net.tcp_listen('0.0.0.0', 4321, opts)
|
148
148
|
|
149
149
|
id = 0
|
150
|
-
|
150
|
+
loop do
|
151
|
+
client = server.accept
|
151
152
|
# log('Accept DF connection', client: client)
|
152
153
|
spin("df#{id += 1}") do
|
153
154
|
Tipi.client_loop(client, {}) { |req| @service.http_request(req, true) }
|
154
155
|
end
|
155
|
-
rescue
|
156
|
-
|
156
|
+
rescue Polyphony::BaseException
|
157
|
+
raise
|
158
|
+
rescue Exception => e
|
159
|
+
log 'DF accept (unknown) error', error: e, backtrace: e.backtrace
|
157
160
|
end
|
158
161
|
end
|
159
162
|
end
|
160
163
|
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
164
|
+
if ENV['TRACE'] == '1'
|
165
|
+
Thread.backend.trace_proc = proc do |event, fiber, value, pri|
|
166
|
+
fiber_id = fiber.tag || fiber.inspect
|
167
|
+
case event
|
168
|
+
when :fiber_schedule
|
169
|
+
log format("=> %s %s %s %s", event, fiber_id, value.inspect, pri ? '(priority)' : '')
|
170
|
+
when :fiber_run
|
171
|
+
log format("=> %s %s %s", event, fiber_id, value.inspect)
|
172
|
+
when :fiber_create, :fiber_terminate
|
173
|
+
log format("=> %s %s", event, fiber_id)
|
174
|
+
else
|
175
|
+
log format("=> %s", event)
|
176
|
+
end
|
177
|
+
end
|
178
|
+
end
|