tipi 0.38 → 0.42

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/test.yml +5 -1
  3. data/.gitignore +5 -0
  4. data/CHANGELOG.md +34 -0
  5. data/Gemfile +5 -1
  6. data/Gemfile.lock +58 -16
  7. data/Rakefile +7 -3
  8. data/TODO.md +77 -1
  9. data/benchmarks/bm_http1_parser.rb +61 -0
  10. data/bin/benchmark +37 -0
  11. data/bin/h1pd +6 -0
  12. data/bin/tipi +3 -21
  13. data/df/sample_agent.rb +1 -1
  14. data/df/server.rb +16 -47
  15. data/df/server_utils.rb +178 -0
  16. data/examples/full_service.rb +13 -0
  17. data/examples/http1_parser.rb +55 -0
  18. data/examples/http_server.rb +15 -3
  19. data/examples/http_server_forked.rb +5 -1
  20. data/examples/http_server_routes.rb +29 -0
  21. data/examples/http_server_static.rb +26 -0
  22. data/examples/http_server_throttled.rb +3 -2
  23. data/examples/https_server.rb +6 -4
  24. data/examples/https_wss_server.rb +2 -1
  25. data/examples/rack_server.rb +5 -0
  26. data/examples/rack_server_https.rb +1 -1
  27. data/examples/rack_server_https_forked.rb +4 -3
  28. data/examples/routing_server.rb +5 -4
  29. data/examples/servername_cb.rb +37 -0
  30. data/examples/websocket_demo.rb +2 -8
  31. data/examples/ws_page.html +2 -2
  32. data/ext/tipi/extconf.rb +13 -0
  33. data/ext/tipi/http1_parser.c +823 -0
  34. data/ext/tipi/http1_parser.h +18 -0
  35. data/ext/tipi/tipi_ext.c +5 -0
  36. data/lib/tipi.rb +89 -1
  37. data/lib/tipi/acme.rb +308 -0
  38. data/lib/tipi/cli.rb +30 -0
  39. data/lib/tipi/digital_fabric/agent.rb +22 -17
  40. data/lib/tipi/digital_fabric/agent_proxy.rb +95 -40
  41. data/lib/tipi/digital_fabric/executive.rb +6 -2
  42. data/lib/tipi/digital_fabric/protocol.rb +87 -15
  43. data/lib/tipi/digital_fabric/request_adapter.rb +6 -10
  44. data/lib/tipi/digital_fabric/service.rb +77 -51
  45. data/lib/tipi/http1_adapter.rb +116 -117
  46. data/lib/tipi/http2_adapter.rb +56 -10
  47. data/lib/tipi/http2_stream.rb +106 -53
  48. data/lib/tipi/rack_adapter.rb +2 -53
  49. data/lib/tipi/response_extensions.rb +17 -0
  50. data/lib/tipi/version.rb +1 -1
  51. data/security/http1.rb +12 -0
  52. data/test/helper.rb +60 -11
  53. data/test/test_http1_parser.rb +586 -0
  54. data/test/test_http_server.rb +0 -27
  55. data/test/test_request.rb +1 -28
  56. data/tipi.gemspec +11 -5
  57. metadata +96 -22
  58. data/e +0 -0
@@ -0,0 +1,178 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'bundler/setup'
4
+ require 'tipi'
5
+ require 'tipi/digital_fabric'
6
+ require 'tipi/digital_fabric/executive'
7
+ require 'json'
8
+ require 'fileutils'
9
+ require 'time'
10
+ require 'polyphony/extensions/debug'
11
+
12
+ FileUtils.cd(__dir__)
13
+
14
+ @service = DigitalFabric::Service.new(token: 'foobar')
15
+ @executive = DigitalFabric::Executive.new(@service, { host: '@executive.realiteq.net' })
16
+
17
+ @pid = Process.pid
18
+
19
+ def log(msg, **ctx)
20
+ text = format(
21
+ "%s (%d) %s\n",
22
+ Time.now.strftime('%Y-%m-%d %H:%M:%S.%3N'),
23
+ @pid,
24
+ msg
25
+ )
26
+ STDOUT.orig_write text
27
+ return if ctx.empty?
28
+
29
+ ctx.each { |k, v| STDOUT.orig_write format(" %s: %s\n", k, v.inspect) }
30
+ end
31
+
32
+ def listen_http
33
+ spin(:http_listener) do
34
+ opts = {
35
+ reuse_addr: true,
36
+ dont_linger: true,
37
+ }
38
+ log('Listening for HTTP on localhost:10080')
39
+ server = Polyphony::Net.tcp_listen('0.0.0.0', 10080, opts)
40
+ id = 0
41
+ loop do
42
+ client = server.accept
43
+ # log("Accept HTTP connection", client: client)
44
+ spin("http#{id += 1}") do
45
+ @service.incr_connection_count
46
+ Tipi.client_loop(client, opts) { |req| @service.http_request(req) }
47
+ ensure
48
+ # log("Done with HTTP connection", client: client)
49
+ @service.decr_connection_count
50
+ end
51
+ rescue Polyphony::BaseException
52
+ raise
53
+ rescue Exception => e
54
+ log 'HTTP accept (unknown) error', error: e, backtrace: e.backtrace
55
+ end
56
+ end
57
+ end
58
+
59
+ CERTIFICATE_REGEXP = /(-----BEGIN CERTIFICATE-----\n[^-]+-----END CERTIFICATE-----\n)/.freeze
60
+
61
+ def listen_https
62
+ spin(:https_listener) do
63
+ private_key = OpenSSL::PKey::RSA.new IO.read('../../reality/ssl/privkey.pem')
64
+ c = IO.read('../../reality/ssl/cacert.pem')
65
+ certificates = c.scan(CERTIFICATE_REGEXP).map { |p| OpenSSL::X509::Certificate.new(p.first) }
66
+ ctx = OpenSSL::SSL::SSLContext.new
67
+ cert = certificates.shift
68
+ log "SSL Certificate expires: #{cert.not_after.inspect}"
69
+ ctx.add_certificate(cert, private_key, certificates)
70
+ ctx.ciphers = 'ECDH+aRSA'
71
+ ctx.send(
72
+ :set_minmax_proto_version,
73
+ OpenSSL::SSL::SSL3_VERSION,
74
+ OpenSSL::SSL::TLS1_3_VERSION
75
+ )
76
+ # ctx.min_version = OpenSSL::SSL::SSL3_VERSION #OpenSSL::SSL::TLS1_VERSION
77
+ # ctx.max_version = OpenSSL::SSL::TLS1_3_VERSION
78
+
79
+ # TODO: further limit ciphers
80
+ # ref: https://github.com/socketry/falcon/blob/3ec805b3ceda0a764a2c5eb68cde33897b6a35ff/lib/falcon/environments/tls.rb
81
+ # ref: https://github.com/socketry/falcon/blob/3ec805b3ceda0a764a2c5eb68cde33897b6a35ff/lib/falcon/tls.rb
82
+
83
+ opts = {
84
+ reuse_addr: true,
85
+ dont_linger: true,
86
+ secure_context: ctx,
87
+ alpn_protocols: Tipi::ALPN_PROTOCOLS
88
+ }
89
+
90
+ log('Listening for HTTPS on localhost:10443')
91
+ server = Polyphony::Net.tcp_listen('0.0.0.0', 10443, opts)
92
+ id = 0
93
+ loop do
94
+ log('Before HTTPS server.accept')
95
+ client = server.accept
96
+ log('After HTTPS server.accept')
97
+ log('Accept HTTPS client connection', client: client)
98
+ spin("https#{id += 1}") do
99
+ @service.incr_connection_count
100
+ Tipi.client_loop(client, opts) { |req| @service.http_request(req) }
101
+ rescue => e
102
+ log('Error while handling HTTPS client', client: client, error: e, backtrace: e.backtrace)
103
+ ensure
104
+ # log("Done with HTTP connection", client: client)
105
+ @service.decr_connection_count
106
+ end
107
+ rescue OpenSSL::SSL::SSLError, SystemCallError, TypeError => e
108
+ log('HTTPS accept error', error: e)
109
+ rescue Polyphony::BaseException
110
+ raise
111
+ rescue Exception => e
112
+ log 'HTTPS accept (unknown) error', error: e, backtrace: e.backtrace
113
+ end
114
+ end
115
+ end
116
+
117
+ UNIX_SOCKET_PATH = '/tmp/df.sock'
118
+ def listen_unix
119
+ spin(:unix_listener) do
120
+ log("Listening on #{UNIX_SOCKET_PATH}")
121
+ FileUtils.rm(UNIX_SOCKET_PATH) if File.exists?(UNIX_SOCKET_PATH)
122
+ socket = UNIXServer.new(UNIX_SOCKET_PATH)
123
+
124
+ id = 0
125
+ loop do
126
+ client = socket.accept
127
+ # log('Accept Unix connection', client: client)
128
+ spin("unix#{id += 1}") do
129
+ Tipi.client_loop(client, {}) { |req| @service.http_request(req, true) }
130
+ end
131
+ rescue Polyphony::BaseException
132
+ raise
133
+ rescue Exception => e
134
+ log 'Unix accept error', error: e, backtrace: e.backtrace
135
+ end
136
+ end
137
+ end
138
+
139
+ def listen_df
140
+ spin(:df_listener) do
141
+ opts = {
142
+ reuse_addr: true,
143
+ reuse_port: true,
144
+ dont_linger: true,
145
+ }
146
+ log('Listening for DF connections on localhost:4321')
147
+ server = Polyphony::Net.tcp_listen('0.0.0.0', 4321, opts)
148
+
149
+ id = 0
150
+ loop do
151
+ client = server.accept
152
+ # log('Accept DF connection', client: client)
153
+ spin("df#{id += 1}") do
154
+ Tipi.client_loop(client, {}) { |req| @service.http_request(req, true) }
155
+ end
156
+ rescue Polyphony::BaseException
157
+ raise
158
+ rescue Exception => e
159
+ log 'DF accept (unknown) error', error: e, backtrace: e.backtrace
160
+ end
161
+ end
162
+ end
163
+
164
+ if ENV['TRACE'] == '1'
165
+ Thread.backend.trace_proc = proc do |event, fiber, value, pri|
166
+ fiber_id = fiber.tag || fiber.inspect
167
+ case event
168
+ when :fiber_schedule
169
+ log format("=> %s %s %s %s", event, fiber_id, value.inspect, pri ? '(priority)' : '')
170
+ when :fiber_run
171
+ log format("=> %s %s %s", event, fiber_id, value.inspect)
172
+ when :fiber_create, :fiber_terminate
173
+ log format("=> %s %s", event, fiber_id)
174
+ else
175
+ log format("=> %s", event)
176
+ end
177
+ end
178
+ end
@@ -0,0 +1,13 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'bundler/setup'
4
+ require 'tipi'
5
+
6
+ ::Exception.__disable_sanitized_backtrace__ = true
7
+
8
+ certificate_db_path = File.expand_path('certificate_store.db', __dir__)
9
+ certificate_store = Tipi::ACME::SQLiteCertificateStore.new(certificate_db_path)
10
+
11
+ Tipi.full_service(
12
+ certificate_store: certificate_store
13
+ ) { |req| req.respond('Hello, world!') }
@@ -0,0 +1,55 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'polyphony'
4
+ require_relative '../lib/tipi_ext'
5
+
6
+ i, o = IO.pipe
7
+
8
+ module ::Kernel
9
+ def trace(*args)
10
+ STDOUT.orig_write(format_trace(args))
11
+ end
12
+
13
+ def format_trace(args)
14
+ if args.first.is_a?(String)
15
+ if args.size > 1
16
+ format("%s: %p\n", args.shift, args)
17
+ else
18
+ format("%s\n", args.first)
19
+ end
20
+ else
21
+ format("%p\n", args.size == 1 ? args.first : args)
22
+ end
23
+ end
24
+ end
25
+
26
+ f = spin do
27
+ parser = Tipi::HTTP1Parser.new(i)
28
+ while true
29
+ trace '*' * 40
30
+ headers = parser.parse_headers
31
+ break unless headers
32
+ trace headers
33
+
34
+ body = parser.read_body
35
+ trace "body: #{body ? body.bytesize : 0} bytes"
36
+ trace body if body && body.bytesize < 80
37
+ end
38
+ end
39
+
40
+ o << "GET /a HTTP/1.1\r\n\r\n"
41
+
42
+ # o << "GET /a HTTP/1.1\r\nContent-Length: 0\r\n\r\n"
43
+
44
+ # o << "GET / HTTP/1.1\r\nHost: localhost:10080\r\nUser-Agent: curl/7.74.0\r\nAccept: */*\r\n\r\n"
45
+
46
+ o << "post /?q=time&blah=blah HTTP/1\r\nTransfer-Encoding: chunked\r\n\r\na\r\nabcdefghij\r\n0\r\n\r\n"
47
+
48
+ data = " " * 4000000
49
+ o << "get /?q=time HTTP/1.1\r\nContent-Length: #{data.bytesize}\r\n\r\n#{data}"
50
+
51
+ o << "get /?q=time HTTP/1.1\r\nCookie: foo\r\nCookie: bar\r\n\r\n"
52
+
53
+ o.close
54
+
55
+ f.await
@@ -9,11 +9,19 @@ opts = {
9
9
  }
10
10
 
11
11
  puts "pid: #{Process.pid}"
12
- puts 'Listening on port 4411...'
12
+ puts 'Listening on port 10080...'
13
+
14
+ # GC.disable
15
+ # Thread.current.backend.idle_gc_period = 60
16
+
17
+ spin_loop(interval: 10) { p Thread.backend.stats }
18
+
19
+ spin_loop(interval: 10) do
20
+ GC.compact
21
+ end
13
22
 
14
23
  spin do
15
- Tipi.serve('0.0.0.0', 4411, opts) do |req|
16
- p path: req.path
24
+ Tipi.serve('0.0.0.0', 10080, opts) do |req|
17
25
  if req.path == '/stream'
18
26
  req.send_headers('Foo' => 'Bar')
19
27
  sleep 1
@@ -21,9 +29,13 @@ spin do
21
29
  sleep 1
22
30
  req.send_chunk("bar\n")
23
31
  req.finish
32
+ elsif req.path == '/upload'
33
+ body = req.read
34
+ req.respond("Body: #{body.inspect} (#{body.bytesize} bytes)")
24
35
  else
25
36
  req.respond("Hello world!\n")
26
37
  end
38
+ # p req.transfer_counts
27
39
  end
28
40
  p 'done...'
29
41
  end.await
@@ -11,11 +11,13 @@ opts = {
11
11
  dont_linger: true
12
12
  }
13
13
 
14
+ server = Tipi.listen('0.0.0.0', 1234, opts)
15
+
14
16
  child_pids = []
15
17
  8.times do
16
18
  pid = Polyphony.fork do
17
19
  puts "forked pid: #{Process.pid}"
18
- Tipi.serve('0.0.0.0', 1234, opts) do |req|
20
+ server.each do |req|
19
21
  req.respond("Hello world! from pid: #{Process.pid}\n")
20
22
  end
21
23
  rescue Interrupt
@@ -25,4 +27,6 @@ end
25
27
 
26
28
  puts 'Listening on port 1234'
27
29
 
30
+ trap('SIGINT') { exit! }
31
+
28
32
  child_pids.each { |pid| Thread.current.backend.waitpid(pid) }
@@ -0,0 +1,29 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'bundler/setup'
4
+ require 'tipi'
5
+
6
+ opts = {
7
+ reuse_addr: true,
8
+ dont_linger: true
9
+ }
10
+
11
+ puts "pid: #{Process.pid}"
12
+ puts 'Listening on port 4411...'
13
+
14
+ app = Tipi.route do |req|
15
+ req.on 'stream' do
16
+ req.send_headers('Foo' => 'Bar')
17
+ sleep 1
18
+ req.send_chunk("foo\n")
19
+ sleep 1
20
+ req.send_chunk("bar\n")
21
+ req.finish
22
+ end
23
+ req.default do
24
+ req.respond("Hello world!\n")
25
+ end
26
+ end
27
+
28
+ trap('INT') { exit! }
29
+ Tipi.serve('0.0.0.0', 4411, opts, &app)
@@ -0,0 +1,26 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'bundler/setup'
4
+ require 'tipi'
5
+ require 'fileutils'
6
+
7
+ opts = {
8
+ reuse_addr: true,
9
+ dont_linger: true
10
+ }
11
+
12
+ puts "pid: #{Process.pid}"
13
+ puts 'Listening on port 4411...'
14
+
15
+ root_path = FileUtils.pwd
16
+
17
+ trap('INT') { exit! }
18
+
19
+ Tipi.serve('0.0.0.0', 4411, opts) do |req|
20
+ path = File.join(root_path, req.path)
21
+ if File.file?(path)
22
+ req.serve_file(path)
23
+ else
24
+ req.respond(nil, ':status' => Qeweney::Status::NOT_FOUND)
25
+ end
26
+ end
@@ -3,9 +3,9 @@
3
3
  require 'bundler/setup'
4
4
  require 'tipi'
5
5
 
6
- $throttler = throttle(1000)
6
+ $throttler = Polyphony::Throttler.new(1000)
7
7
  opts = { reuse_addr: true, dont_linger: true }
8
- spin do
8
+ server = spin do
9
9
  Tipi.serve('0.0.0.0', 1234, opts) do |req|
10
10
  $throttler.call { req.respond("Hello world!\n") }
11
11
  end
@@ -13,3 +13,4 @@ end
13
13
 
14
14
  puts "pid: #{Process.pid}"
15
15
  puts 'Listening on port 1234...'
16
+ server.await
@@ -19,11 +19,13 @@ Tipi.serve('0.0.0.0', 1234, opts) do |req|
19
19
  p path: req.path
20
20
  if req.path == '/stream'
21
21
  req.send_headers('Foo' => 'Bar')
22
- sleep 1
22
+ sleep 0.5
23
23
  req.send_chunk("foo\n")
24
- sleep 1
25
- req.send_chunk("bar\n")
26
- req.finish
24
+ sleep 0.5
25
+ req.send_chunk("bar\n", done: true)
26
+ elsif req.path == '/upload'
27
+ body = req.read
28
+ req.respond("Body: #{body.inspect} (#{body.bytesize} bytes)")
27
29
  else
28
30
  req.respond("Hello world!\n")
29
31
  end
@@ -2,6 +2,7 @@
2
2
 
3
3
  require 'bundler/setup'
4
4
  require 'tipi'
5
+ require 'tipi/websocket'
5
6
  require 'localhost/authority'
6
7
 
7
8
  def ws_handler(conn)
@@ -26,7 +27,7 @@ opts = {
26
27
  dont_linger: true,
27
28
  secure_context: authority.server_context,
28
29
  upgrade: {
29
- websocket: Polyphony::Websocket.handler(&method(:ws_handler))
30
+ websocket: Tipi::Websocket.handler(&method(:ws_handler))
30
31
  }
31
32
  }
32
33
 
@@ -4,6 +4,11 @@ require 'bundler/setup'
4
4
  require 'tipi'
5
5
 
6
6
  app_path = ARGV.first || File.expand_path('./config.ru', __dir__)
7
+ unless File.file?(app_path)
8
+ STDERR.puts "Please provide rack config file (there are some in the examples directory.)"
9
+ exit!
10
+ end
11
+
7
12
  app = Tipi::RackAdapter.load(app_path)
8
13
  opts = { reuse_addr: true, dont_linger: true }
9
14
 
@@ -5,7 +5,7 @@ require 'tipi'
5
5
  require 'localhost/authority'
6
6
 
7
7
  app_path = ARGV.first || File.expand_path('./config.ru', __dir__)
8
- app = Polyphony::HTTP::Server::RackAdapter.load(app_path)
8
+ app = Tipi::RackAdapter.load(app_path)
9
9
 
10
10
  authority = Localhost::Authority.fetch
11
11
  opts = {
@@ -5,15 +5,16 @@ require 'tipi'
5
5
  require 'localhost/authority'
6
6
 
7
7
  app_path = ARGV.first || File.expand_path('./config.ru', __dir__)
8
- app = Polyphony::HTTP::Server::RackAdapter.load(app_path)
8
+ app = Tipi::RackAdapter.load(app_path)
9
9
 
10
10
  authority = Localhost::Authority.fetch
11
11
  opts = {
12
12
  reuse_addr: true,
13
+ reuse_port: true,
13
14
  dont_linger: true,
14
15
  secure_context: authority.server_context
15
16
  }
16
- server = Polyphony::HTTP::Server.listen('0.0.0.0', 1234, opts)
17
+ server = Tipi.listen('0.0.0.0', 1234, opts)
17
18
  puts 'Listening on port 1234'
18
19
 
19
20
  child_pids = []
@@ -24,4 +25,4 @@ child_pids = []
24
25
  end
25
26
  end
26
27
 
27
- child_pids.each { |pid| EV::Child.new(pid).await }
28
+ child_pids.each { |pid| Thread.current.backend.waitpid(pid) }