tipi 0.38 → 0.39

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6a8c075ce0b769f014a20587dd061f17e32361f211aad5741f655dafe6687e37
-  data.tar.gz: d14c0fe026a7db1aa46edded54e5ee6f1a0f8598f626dd1aff342ad07b5cec39
+  metadata.gz: 02c5ce4798e1961ca59798b75bafe158570e777661f6fc8667169553df3cd3f4
+  data.tar.gz: 405bfa3526efaad394d34840764dc131230a366574cfc52b63344a6d5cdfe6dd
 SHA512:
-  metadata.gz: e306783fc83d9d7ebd0678c4d68a13f6f9bd77e4b9d83fc84d745e5cc3f53eb27b69286fb1ba9f1ca54fe57358009a4c11dc9374c8a8d9046dc4330e359dc988
-  data.tar.gz: 789c3b4e1374cc57e04e3bce4aa8d12022cfd7e31c9cba4f5777302f57943da0a967f17f8cb2f91551357deac1bbd07fe514c2cb2e002331f525d341c6b7f0e9
+  metadata.gz: 754bd35136e4e004e6bd782eb2060d933b2439c437a9c2a966529e0465c42096b97fa08b1dae04b7a3c8aa0c77c367fd5395d215cca73fd7cf1aa80c297b3178
+  data.tar.gz: 24425d798c076ad43b4fa45b641a90a5e801dfe35acd6217e8a8fc97ea546f18457fc9a8653e47f7bd5f696da885ef86db8db36385dcaa79ccbcf9d42d64081c

data/.github/workflows/test.yml

@@ -22,6 +22,6 @@ jobs:
     - name: Install dependencies
       run: |
         gem install bundler
-        bundle install
+        POLYPHONY_USE_LIBEV=1 bundle install
     - name: Run tests
       run:  bundle exec rake test

data/.gitignore

@@ -54,3 +54,4 @@ build-iPhoneSimulator/
 
 # Used by RuboCop. Remote config files pulled in from inherit_from directive.
 # .rubocop-https?--*
+log

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 0.39 2021-06-20
+
+- More work on DF server
+- Fix HTTP2StreamHandler#send_headers
+- Various fixes to HTTP/2 adapter
+- Fix host detection for HTTP/2 connections
+- Fix HTTP/1 adapter #respond with nil body
+- Fix HTTP1Adapter#send_headers
+
 ## 0.38 2021-03-09
 
 - Don't use chunked transfer encoding for non-streaming responses

data/Gemfile.lock

@@ -1,12 +1,12 @@
 PATH
   remote: .
   specs:
-    tipi (0.38)
+    tipi (0.39)
       http-2 (~> 0.10.0)
       http_parser.rb (~> 0.6.0)
       msgpack (~> 1.4.2)
-      polyphony (~> 0.52.0)
-      qeweney (~> 0.6)
+      polyphony (~> 0.55.0)
+      qeweney (~> 0.9.1)
       rack (>= 2.0.8, < 2.3.0)
       websocket (~> 1.2.8)
 
@@ -28,8 +28,8 @@ GEM
       minitest (>= 5.0)
       ruby-progressbar
     msgpack (1.4.2)
-    polyphony (0.52.0)
-    qeweney (0.7.5)
+    polyphony (0.55.0)
+    qeweney (0.9.1)
       escape_utils (~> 1.2.1)
     rack (2.2.3)
     rake (12.3.3)

data/TODO.md

@@ -1,4 +1,80 @@
-For immediate execution:
+## Add an API for reading a request body chunk into an IO (pipe)
+
+      ```ruby
+      # currently
+      chunk = req.next_chunk
+      # or
+      req.each_chunk { |c| do_something(c) }
+
+      # what we'd like to do
+      r, w = IO.pipe
+      len = req.splice_chunk(w)
+      sock << "Here comes a chunk of #{len} bytes\n"
+      sock.splice(r, len)
+
+      # or:
+      r, w = IO.pipe
+      req.splice_each_chunk(w) do |len|
+        sock << "Here comes a chunk of #{len} bytes\n"
+        sock.splice(r, len)
+      end
+      ```
+
+# HTTP/1.1 parser
+
+- httparser.rb is not actively updated
+- the httparser.rb C parser code comes originally from https://github.com/nodejs/llhttp
+- there's a Ruby gem https://github.com/metabahn/llhttp, but its API is too low-level
+  (lots of callbacks, headers need to be retained across callbacks)
+- the basic idea is to import the C-code, then build a parser object with the following
+  callbacks:
+
+  ```ruby
+  on_headers_complete(headers)
+  on_body_chunk(chunk)
+  on_message_complete
+  ```
+
+- The llhttp gem's C-code is here: https://github.com/metabahn/llhttp/tree/main/mri
+
+- Actually, if you do a C extension, instead of a callback-based API, we can
+  design a blocking API:
+
+  ```ruby
+  parser = Tipi::HTTP1::Parser.new
+  parser.each_request(socket) do |headers|
+    request = Request.new(normalize_headers(headers))
+    handle_request(request)
+  end
+  ```
+
+# What about HTTP/2?
+
+It would be a nice exercise in converting a callback-based API to a blocking
+one:
+
+```ruby
+parser = Tipi::HTTP2::Parser.new(socket)
+parser.each_stream(socket) do |stream|
+  spin { handle_stream(stream) }
+end
+```
+
+
+
+# DF
+
+- Add attack protection for IP-address HTTP host:
+
+  ```ruby
+  IPV4_REGEXP = /^\d+\.\d+\.\d+\.\d+$/.freeze
+
+  def is_attack_request?(req)
+    return true if req.host =~ IPV4_REGEXP && req.query[:q] != 'ping'
+  end
+  ```
+
+- Add attack route to Qeweney routing API
 
 
 

data/df/sample_agent.rb

@@ -13,7 +13,7 @@ class SampleAgent < DigitalFabric::Agent
   HTML_SSE = IO.read(File.join(__dir__, 'sse_page.html'))
 
   def http_request(req)
-    path = req['headers'][':path']
+    path = req.headers[':path']
     case path
     when '/agent'
       send_df_message(Protocol.http_response(

data/df/server.rb

@@ -6,6 +6,8 @@ require 'tipi/digital_fabric'
 require 'tipi/digital_fabric/executive'
 require 'json'
 require 'fileutils'
+require 'localhost/authority'
+
 FileUtils.cd(__dir__)
 
 service = DigitalFabric::Service.new(token: 'foobar')
@@ -19,13 +21,46 @@ end
 
 puts "pid: #{Process.pid}"
 
-tcp_listener = spin do
+http_listener = spin do
   opts = {
     reuse_addr:  true,
     dont_linger: true,
   }
-  puts 'Listening on localhost:4411'
-  server = Polyphony::Net.tcp_listen('0.0.0.0', 4411, opts)
+  puts 'Listening for HTTP on localhost:10080'
+  server = Polyphony::Net.tcp_listen('0.0.0.0', 10080, opts)
+  server.accept_loop do |client|
+    spin do
+      service.incr_connection_count
+      Tipi.client_loop(client, opts) { |req| service.http_request(req) }
+    ensure
+      service.decr_connection_count
+    end
+  rescue Exception => e
+    puts "HTTP accept_loop error: #{e.inspect}"
+    puts e.backtrace.join("\n")
+  end
+end
+
+CERTIFICATE_REGEXP = /(-----BEGIN CERTIFICATE-----\n[^-]+-----END CERTIFICATE-----\n)/.freeze
+
+https_listener = spin do
+  private_key = OpenSSL::PKey::RSA.new IO.read('../../reality/ssl/privkey.pem')
+  c = IO.read('../../reality/ssl/cacert.pem')
+  certificates = c.scan(CERTIFICATE_REGEXP).map { |p|  OpenSSL::X509::Certificate.new(p.first) }
+  ctx = OpenSSL::SSL::SSLContext.new
+  cert = certificates.shift
+  puts "Certificate expires: #{cert.not_after.inspect}"
+  ctx.add_certificate(cert, private_key, certificates)
+  # ctx = Localhost::Authority.fetch.server_context
+  opts = {
+    reuse_addr:     true,
+    dont_linger:    true,
+    secure_context: ctx,
+    alpn_protocols: Tipi::ALPN_PROTOCOLS
+  }
+
+  puts 'Listening for HTTPS on localhost:10443'
+  server = Polyphony::Net.tcp_listen('0.0.0.0', 10443, opts)
   server.accept_loop do |client|
     spin do
       service.incr_connection_count
@@ -33,6 +68,9 @@ tcp_listener = spin do
     ensure
       service.decr_connection_count
     end
+  rescue Exception => e
+    puts "HTTPS accept_loop error: #{e.inspect}"
+    puts e.backtrace.join("\n")
   end
 end
 
@@ -46,9 +84,13 @@ unix_listener = spin do
 end
 
 begin
-  Fiber.await(tcp_listener, unix_listener)
+  Fiber.await(http_listener, https_listener, unix_listener)
 rescue Interrupt
   puts "Got SIGINT, shutting down gracefully"
   service.graceful_shutdown
   puts "post graceful shutdown"
+rescue Exception => e
+  puts '*' * 40
+  p e
+  puts e.backtrace.join("\n")
 end

data/examples/http_server.rb

@@ -13,7 +13,6 @@ puts 'Listening on port 4411...'
 
 spin do
   Tipi.serve('0.0.0.0', 4411, opts) do |req|
-    p path: req.path
     if req.path == '/stream'
       req.send_headers('Foo' => 'Bar')
       sleep 1
@@ -24,6 +23,7 @@ spin do
     else
       req.respond("Hello world!\n")
     end
+    p req.transfer_counts
   end
   p 'done...'
 end.await

data/examples/http_server_forked.rb

@@ -25,4 +25,6 @@ end
 
 puts 'Listening on port 1234'
 
+trap('SIGINT') { exit! }
+
 child_pids.each { |pid| Thread.current.backend.waitpid(pid) }

data/examples/http_server_throttled.rb

@@ -3,9 +3,9 @@
 require 'bundler/setup'
 require 'tipi'
 
-$throttler = throttle(1000)
+$throttler = Polyphony::Throttler.new(1000)
 opts = { reuse_addr: true, dont_linger: true }
-spin do
+server = spin do
   Tipi.serve('0.0.0.0', 1234, opts) do |req|
     $throttler.call { req.respond("Hello world!\n") }
   end
@@ -13,3 +13,4 @@ end
 
 puts "pid: #{Process.pid}"
 puts 'Listening on port 1234...'
+server.await

data/examples/https_server.rb

@@ -19,11 +19,10 @@ Tipi.serve('0.0.0.0', 1234, opts) do |req|
   p path: req.path
   if req.path == '/stream'
     req.send_headers('Foo' => 'Bar')
-    sleep 1
+    sleep 0.5
     req.send_chunk("foo\n")
-    sleep 1
-    req.send_chunk("bar\n")
-    req.finish
+    sleep 0.5
+    req.send_chunk("bar\n", done: true)
   else
     req.respond("Hello world!\n")
   end

data/examples/https_wss_server.rb

@@ -2,6 +2,7 @@
 
 require 'bundler/setup'
 require 'tipi'
+require 'tipi/websocket'
 require 'localhost/authority'
 
 def ws_handler(conn)
@@ -26,7 +27,7 @@ opts = {
   dont_linger:    true,
   secure_context: authority.server_context,
   upgrade:        {
-    websocket: Polyphony::Websocket.handler(&method(:ws_handler))
+    websocket: Tipi::Websocket.handler(&method(:ws_handler))
   }
 }
 

data/examples/rack_server.rb

@@ -4,6 +4,11 @@ require 'bundler/setup'
 require 'tipi'
 
 app_path = ARGV.first || File.expand_path('./config.ru', __dir__)
+unless File.file?(app_path)
+  STDERR.puts "Please provide rack config file (there are some in the examples directory.)"
+  exit!
+end
+
 app = Tipi::RackAdapter.load(app_path)
 opts = { reuse_addr: true, dont_linger: true }
 

data/examples/rack_server_https.rb

@@ -5,7 +5,7 @@ require 'tipi'
 require 'localhost/authority'
 
 app_path = ARGV.first || File.expand_path('./config.ru', __dir__)
-app = Polyphony::HTTP::Server::RackAdapter.load(app_path)
+app = Tipi::RackAdapter.load(app_path)
 
 authority = Localhost::Authority.fetch
 opts = {

data/examples/rack_server_https_forked.rb

@@ -5,15 +5,16 @@ require 'tipi'
 require 'localhost/authority'
 
 app_path = ARGV.first || File.expand_path('./config.ru', __dir__)
-app = Polyphony::HTTP::Server::RackAdapter.load(app_path)
+app = Tipi::RackAdapter.load(app_path)
 
 authority = Localhost::Authority.fetch
 opts = {
   reuse_addr:     true,
+  reuse_port:     true,
   dont_linger:    true,
   secure_context: authority.server_context
 }
-server = Polyphony::HTTP::Server.listen('0.0.0.0', 1234, opts)
+server = Tipi.listen('0.0.0.0', 1234, opts)
 puts 'Listening on port 1234'
 
 child_pids = []
@@ -24,4 +25,4 @@ child_pids = []
   end
 end
 
-child_pids.each { |pid| EV::Child.new(pid).await }
+child_pids.each { |pid| Thread.current.backend.waitpid(pid) }

data/examples/routing_server.rb

@@ -12,17 +12,18 @@ puts "pid: #{Process.pid}"
 puts 'Listening on port 4411...'
 
 app = Tipi.route do |r|
-  r.root do
+
+  r.on_root do
     r.redirect '/hello'
   end
   r.on 'hello' do
-    r.get 'world' do
+    r.on_get 'world' do
       r.respond 'Hello world'
     end
-    r.get do
+    r.on_get do
       r.respond 'Hello'
     end
-    r.post do
+    r.on_post do
       puts 'Someone said Hello'
       r.redirect '/'
     end

data/lib/tipi/digital_fabric/agent.rb

@@ -119,7 +119,7 @@ module DigitalFabric
 
     def recv_df_message(msg)
       @last_recv = Time.now
-      case msg['kind']
+      case msg[Protocol::Attribute::KIND]
       when Protocol::SHUTDOWN
         recv_shutdown
       when Protocol::HTTP_REQUEST
@@ -130,7 +130,7 @@ module DigitalFabric
         recv_ws_request(msg)
       when Protocol::CONN_DATA, Protocol::CONN_CLOSE,
            Protocol::WS_DATA, Protocol::WS_CLOSE
-        fiber = @requests[msg['id']]
+        fiber = @requests[msg[Protocol::Attribute::ID]]
         fiber << msg if fiber
       end
     end
@@ -140,7 +140,7 @@ module DigitalFabric
       # messages. This is so we can correctly stop long-running requests
       # upon graceful shutdown
       if is_long_running_request_response?(msg)
-        id = msg[:id]
+        id = msg[Protocol::Attribute::ID]
         @long_running_requests[id] = @requests[id]
       end
       @last_send = Time.now
@@ -148,11 +148,11 @@ module DigitalFabric
     end
 
     def is_long_running_request_response?(msg)
-      case msg[:kind]
+      case msg[Protocol::Attribute::KIND]
       when Protocol::HTTP_UPGRADE
         true
       when Protocol::HTTP_RESPONSE
-        msg[:body] && !msg[:complete]
+        !msg[Protocol::Attribute::HttpResponse::COMPLETE]
       end
     end
 
@@ -165,7 +165,7 @@ module DigitalFabric
 
     def recv_http_request(msg)
       req = prepare_http_request(msg)
-      id = msg['id']
+      id = msg[Protocol::Attribute::ID]
       @requests[id] = spin do
         http_request(req)
       rescue IOError, Errno::ECONNREFUSED, Errno::EPIPE
@@ -180,17 +180,20 @@ module DigitalFabric
     end
 
     def prepare_http_request(msg)
-      req = Qeweney::Request.new(msg['headers'], RequestAdapter.new(self, msg))
-      req.buffer_body_chunk(msg['body']) if msg['body']
-      req.complete! if msg['complete']
+      headers = msg[Protocol::Attribute::HttpRequest::HEADERS]
+      body_chunk = msg[Protocol::Attribute::HttpRequest::BODY_CHUNK]
+      complete = msg[Protocol::Attribute::HttpRequest::COMPLETE]
+      req = Qeweney::Request.new(headers, RequestAdapter.new(self, msg))
+      req.buffer_body_chunk(body_chunk) if body_chunk
+      req.complete! if complete
       req
     end
 
     def recv_http_request_body(msg)
-      fiber = @requests[msg['id']]
+      fiber = @requests[msg[Protocol::Attribute::ID]]
       return unless fiber
 
-      fiber << msg['body']
+      fiber << msg[Protocol::Attribute::HttpRequestBody::BODY]
     end
 
     def get_http_request_body(id, limit)
@@ -199,8 +202,8 @@ module DigitalFabric
     end
 
     def recv_ws_request(msg)
-      req = Qeweney::Request.new(msg['headers'], RequestAdapter.new(self, msg))
-      id = msg['id']
+      req = Qeweney::Request.new(msg[Protocol::Attribute::WS::HEADERS], RequestAdapter.new(self, msg))
+      id = msg[Protocol::Attribute::ID]
       @requests[id] = @long_running_requests[id] = spin do
         ws_request(req)
       rescue IOError, Errno::ECONNREFUSED, Errno::EPIPE