tipi 0.41 → 0.42

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5a83c181c05d8d8c8de50f236feab65b800b1bd266c2f0a4d4696d4853ac39fd
-  data.tar.gz: d9a84fc9f72c3a5565f587e35fb4ed206c83c863fbb0b4b470449919a687463b
+  metadata.gz: 82155f4b86223d3fb32dc8c314abfeb75fda951747e63832ed5c15bc3b1f43cc
+  data.tar.gz: 721b98fb8d330d1bc38f8f236f82ddd909235d31f5875c1b4f89895e4e3926da
 SHA512:
-  metadata.gz: e9c13f387c3431888d9d43c88d5bd00831c4e2e8976bf8ae4e1acc479b8d50fa888cd58af0fc433da2dab8e3ea09b99ca0adbdcc61abc71457a805c26aa03687
-  data.tar.gz: df01a54c7c46928c67da321cfc830d8f33e8af7fa8197cd5930f3de73838efd4849d51b10af35b1a5297f7f484b296ff2ed6146dbb8d66476bf0b714f1dbfd68
+  metadata.gz: 10389779975234d90c968626b4f9d6b168491f9a32ecf1be71683c37cf11f58f1bac52a36478691e72328236a16d309a8462104bda556772e25cebe52b2b0d53
+  data.tar.gz: 99db2c15278cdfc286ffad05f266f5d62eec6a3bb1363f8dfa4c302549a308a691496ee60ec085ede59e3b594533e8e196bbca144878e5a6dccf2ecfc962151c

data/.github/workflows/test.yml

@@ -23,5 +23,9 @@ jobs:
       run: |
         gem install bundler
         POLYPHONY_USE_LIBEV=1 bundle install
+    - name: Show Linux kernel version
+      run: uname -r
+    - name: Compile C-extension
+      run: bundle exec rake compile
     - name: Run tests
       run:  bundle exec rake test

data/.gitignore

@@ -55,5 +55,7 @@ build-iPhoneSimulator/
 # Used by RuboCop. Remote config files pulled in from inherit_from directive.
 # .rubocop-https?--*
 log
+log.*
 
-lib/tipi_ext*
+lib/tipi_ext*
+examples/certificate_store.db

data/CHANGELOG.md

@@ -1,3 +1,14 @@
+## 0.42 2021-08-16
+
+- HTTP/1 parser:  disable UTF-8 parsing for all but header values
+- Add support for parsing HTTP/1 from callable source
+- Introduce full_service API for automatic HTTPS
+- Introduce automatic SSL certificate provisioning
+- Improve handling of exceptions
+- Various fixes to DF service and agent pxoy
+- Fix upgrading to HTTP2 with a request body
+- Switch to new HTTP/1 parser
+
 ## 0.41 2021-07-26
 
 - Fix Rack adapter (#11)

data/Gemfile

@@ -1,3 +1,7 @@
 source 'https://rubygems.org'
 
-gemspec
+gemspec
+%w{polyphony qeweney}.each do |dep|
+  dir = "../#{dep}"
+  gem(dep, path: dir) if File.directory?(dir)
+end

data/Gemfile.lock

@@ -1,13 +1,25 @@
+PATH
+  remote: ../polyphony
+  specs:
+    polyphony (0.69)
+
+PATH
+  remote: ../qeweney
+  specs:
+    qeweney (0.14)
+      escape_utils (~> 1.2.1)
+
 PATH
   remote: .
   specs:
-    tipi (0.41)
+    tipi (0.42)
       acme-client (~> 2.0.8)
-      http-2 (~> 0.10.0)
-      http_parser.rb (~> 0.6.0)
+      extralite (~> 1.2)
+      http-2 (~> 0.11)
+      localhost (~> 1.1.4)
       msgpack (~> 1.4.2)
-      polyphony (~> 0.64)
-      qeweney (~> 0.11)
+      polyphony (~> 0.69)
+      qeweney (~> 0.14)
       rack (>= 2.0.8, < 2.3.0)
       websocket (~> 1.2.8)
 
@@ -22,7 +34,8 @@ GEM
       rack (>= 1.6.0)
     docile (1.4.0)
     escape_utils (1.2.1)
-    faraday (1.5.1)
+    extralite (1.2)
+    faraday (1.7.0)
       faraday-em_http (~> 1.0)
       faraday-em_synchrony (~> 1.0)
       faraday-excon (~> 1.1)
@@ -30,6 +43,7 @@ GEM
       faraday-net_http (~> 1.0)
       faraday-net_http_persistent (~> 1.1)
       faraday-patron (~> 1.0)
+      faraday-rack (~> 1.0)
       multipart-post (>= 1.2, < 3)
       ruby2_keywords (>= 0.0.4)
     faraday-em_http (1.0.0)
@@ -39,8 +53,9 @@ GEM
     faraday-net_http (1.0.1)
     faraday-net_http_persistent (1.2.0)
     faraday-patron (1.0.0)
-    http-2 (0.10.2)
-    http_parser.rb (0.6.0)
+    faraday-rack (1.0.0)
+    http-2 (0.11.0)
+    http_parser.rb (0.7.0)
     json (2.5.1)
     localhost (1.1.8)
     minitest (5.11.3)
@@ -51,9 +66,6 @@ GEM
       ruby-progressbar
     msgpack (1.4.2)
     multipart-post (2.1.1)
-    polyphony (0.64)
-    qeweney (0.11)
-      escape_utils (~> 1.2.1)
     rack (2.2.3)
     rake (12.3.3)
     rake-compiler (1.1.1)
@@ -72,9 +84,11 @@ PLATFORMS
 
 DEPENDENCIES
   cuba (~> 3.9.3)
-  localhost (~> 1.1.4)
+  http_parser.rb (= 0.7.0)
   minitest (~> 5.11.3)
   minitest-reporters (~> 1.4.2)
+  polyphony!
+  qeweney!
   rake (~> 12.3.3)
   rake-compiler (= 1.1.1)
   simplecov (~> 0.17.1)

data/benchmarks/bm_http1_parser.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+
+HTTP_REQUEST = "GET /foo HTTP/1.1\r\nHost: example.com\r\nAccept: */*\r\n\r\n"
+
+def benchmark_other_http1_parser(iterations)
+  STDOUT << "http_parser.rb: "
+  require 'http_parser.rb'
+  
+  i, o = IO.pipe
+  parser = Http::Parser.new
+  done = false
+  headers = nil
+  parser.on_headers_complete = proc do |h|
+    headers = h
+    headers[':method'] = parser.http_method
+    headers[':path'] = parser.request_url
+    headers[':protocol'] = parser.http_version
+  end
+  parser.on_message_complete = proc { done = true }
+
+  t0 = Time.now
+  iterations.times do
+    o << HTTP_REQUEST
+    done = false
+    while !done
+      msg = i.readpartial(4096)
+      parser << msg
+    end
+  end
+  t1 = Time.now
+  puts "#{iterations / (t1 - t0)} ips"
+end
+
+def benchmark_tipi_http1_parser(iterations)
+  STDOUT << "tipi parser: "
+  require_relative '../lib/tipi_ext'
+  i, o = IO.pipe
+  reader = proc { |len| i.readpartial(len) }
+  parser = Tipi::HTTP1Parser.new(reader)
+
+  t0 = Time.now
+  iterations.times do
+    o << HTTP_REQUEST
+    headers = parser.parse_headers
+  end
+  t1 = Time.now
+  puts "#{iterations / (t1 - t0)} ips"
+end
+
+def fork_benchmark(method, iterations)
+  pid = fork { send(method, iterations) }
+  Process.wait(pid)
+end
+
+x = 500000
+fork_benchmark(:benchmark_other_http1_parser, x)
+fork_benchmark(:benchmark_tipi_http1_parser, x)
+
+# benchmark_tipi_http1_parser(x)

data/bin/benchmark

@@ -0,0 +1,37 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'polyphony'
+
+def parse_latency(latency)
+  m = latency.match(/^([\d\.]+)(us|ms|s)$/)
+  return nil unless m
+
+  value = m[1].to_f
+  case m[2]
+  when 's' then value
+  when 'ms' then value / 1000
+  when 'us' then value / 1000000
+  end
+end
+
+def parse_wrk_results(results)
+  lines = results.lines
+  latencies = lines[3].strip.split(/\s+/)
+  throughput = lines[6].strip.split(/\s+/)
+
+  {
+    latency_avg:  parse_latency(latencies[1]),
+    latency_max:  parse_latency(latencies[3]),
+    rate:         throughput[1].to_f
+  }
+end
+
+def run_wrk(duration: 10, threads: 2, connections: 10, url: )
+  `wrk -d#{duration} -t#{threads} -c#{connections} #{url}`
+end
+
+[8, 64, 256, 512].each do |c|
+  puts "connections: #{c}"
+  p parse_wrk_results(run_wrk(duration: 10, threads: 4, connections: c, url: "http://localhost:10080/"))
+end

data/bin/h1pd

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -e
+rake compile
+ruby test/test_http1_parser.rb
+ruby benchmarks/bm_http1_parser.rb

data/bin/tipi

@@ -1,26 +1,8 @@
 #!/usr/bin/env ruby
 
 require 'bundler/setup'
-require 'polyphony'
-require File.expand_path('../lib/tipi/configuration', __dir__)
+require 'tipi/cli'
 
-config = {}
-#config[:forked] = 4
+trap('SIGINT') { exit }
 
-puts DATA.read
-puts
-
-configuration_manager = spin { Tipi::Configuration.supervise_config }
-
-configuration_manager << config
-configuration_manager.await
-
-__END__
-
-          ooo
-       oo
-     o
-   \|/
-   / \   Tipi - A better web server for a better world
-  /___\
-  
+Tipi::CLI.start

data/df/server.rb

@@ -17,7 +17,7 @@ rescue Interrupt
 rescue SystemExit
   # ignore
 rescue Exception => e
-  log("Uncaught exception", error: e, backtrace: e.backtrace)
+  log("Uncaught exception", error: e, source: e.source_fiber, raising: e.raising_fiber, backtrace: e.backtrace)
 ensure
   log('DF server stopped')
 end

data/df/server_utils.rb

@@ -7,24 +7,7 @@ require 'tipi/digital_fabric/executive'
 require 'json'
 require 'fileutils'
 require 'time'
-
-module ::Kernel
-  def trace(*args)
-    STDOUT.orig_write(format_trace(args))
-  end
-
-  def format_trace(args)
-    if args.first.is_a?(String)
-      if args.size > 1
-        format("%s: %p\n", args.shift, args)
-      else
-        format("%s\n", args.first)
-      end
-    else
-      format("%p\n", args.size == 1 ? args.first : args)
-    end
-  end
-end
+require 'polyphony/extensions/debug'
 
 FileUtils.cd(__dir__)
 
@@ -65,8 +48,10 @@ def listen_http
         # log("Done with HTTP connection", client: client)
         @service.decr_connection_count
       end
-    rescue => e
-      log("HTTP accept loop error", error: e, backtrace: e.backtrace)
+    rescue Polyphony::BaseException
+      raise
+    rescue Exception => e
+      log 'HTTP accept (unknown) error', error: e, backtrace: e.backtrace
     end
   end
 end
@@ -74,7 +59,7 @@ end
 CERTIFICATE_REGEXP = /(-----BEGIN CERTIFICATE-----\n[^-]+-----END CERTIFICATE-----\n)/.freeze
 
 def listen_https
-  spin('https_listener') do
+  spin(:https_listener) do
     private_key = OpenSSL::PKey::RSA.new IO.read('../../reality/ssl/privkey.pem')
     c = IO.read('../../reality/ssl/cacert.pem')
     certificates = c.scan(CERTIFICATE_REGEXP).map { |p|  OpenSSL::X509::Certificate.new(p.first) }
@@ -83,6 +68,13 @@ def listen_https
     log "SSL Certificate expires: #{cert.not_after.inspect}"
     ctx.add_certificate(cert, private_key, certificates)
     ctx.ciphers = 'ECDH+aRSA'
+    ctx.send(
+      :set_minmax_proto_version,
+      OpenSSL::SSL::SSL3_VERSION,
+      OpenSSL::SSL::TLS1_3_VERSION
+    )
+    # ctx.min_version = OpenSSL::SSL::SSL3_VERSION #OpenSSL::SSL::TLS1_VERSION
+    # ctx.max_version = OpenSSL::SSL::TLS1_3_VERSION
 
     # TODO: further limit ciphers
     # ref: https://github.com/socketry/falcon/blob/3ec805b3ceda0a764a2c5eb68cde33897b6a35ff/lib/falcon/environments/tls.rb
@@ -99,8 +91,10 @@ def listen_https
     server = Polyphony::Net.tcp_listen('0.0.0.0', 10443, opts)
     id = 0
     loop do
+      log('Before HTTPS server.accept')
       client = server.accept
-      # log('Accept HTTPS client connection', client: client)
+      log('After HTTPS server.accept')
+      log('Accept HTTPS client connection', client: client)
       spin("https#{id += 1}") do
         @service.incr_connection_count
         Tipi.client_loop(client, opts) { |req| @service.http_request(req) }
@@ -111,9 +105,11 @@ def listen_https
         @service.decr_connection_count
       end
     rescue OpenSSL::SSL::SSLError, SystemCallError, TypeError => e
-      # log('HTTPS accept error', error: e, backtrace: e.backtrace)
-    rescue => e
-      log('HTTPS accept (unknown) error', error: e, backtrace: e.backtrace)
+      log('HTTPS accept error', error: e)
+    rescue Polyphony::BaseException
+      raise
+    rescue Exception => e
+      log 'HTTPS accept (unknown) error', error: e, backtrace: e.backtrace
     end
   end
 end
@@ -126,13 +122,16 @@ def listen_unix
     socket = UNIXServer.new(UNIX_SOCKET_PATH)
 
     id = 0
-    socket.accept_loop do |client|
+    loop do
+      client = socket.accept
       # log('Accept Unix connection', client: client)
       spin("unix#{id += 1}") do
         Tipi.client_loop(client, {}) { |req| @service.http_request(req, true) }
       end
-    rescue OpenSSL::SSL::SSLError
-      # disregard
+    rescue Polyphony::BaseException
+      raise
+    rescue Exception => e
+      log 'Unix accept error', error: e, backtrace: e.backtrace
     end
   end
 end
@@ -141,33 +140,39 @@ def listen_df
   spin(:df_listener) do
     opts = {
       reuse_addr:  true,
+      reuse_port:  true,
       dont_linger: true,
     }
     log('Listening for DF connections on localhost:4321')
     server = Polyphony::Net.tcp_listen('0.0.0.0', 4321, opts)
 
     id = 0
-    server.accept_loop do |client|
+    loop do
+      client = server.accept
       # log('Accept DF connection', client: client)
       spin("df#{id += 1}") do
         Tipi.client_loop(client, {}) { |req| @service.http_request(req, true) }
       end
-    rescue OpenSSL::SSL::SSLError
-      # disregard
+    rescue Polyphony::BaseException
+      raise
+    rescue Exception => e
+      log 'DF accept (unknown) error', error: e, backtrace: e.backtrace
     end
   end
 end
 
-# Thread.backend.trace_proc = proc do |event, fiber, value, pri|
-#   fiber_id = fiber.tag || fiber.inspect
-#   case event
-#   when :fiber_schedule
-#     log format("=> %s %s %s %s", event, fiber_id, value.inspect, pri ? '' : '(priority)')
-#   when :fiber_run
-#     log format("=> %s %s %s", event, fiber_id, value.inspect)
-#   when :fiber_create, :fiber_terminate
-#     log format("=> %s %s", event, fiber_id)
-#   else
-#     log format("=> %s", event)
-#   end
-# end
+if ENV['TRACE'] == '1'
+  Thread.backend.trace_proc = proc do |event, fiber, value, pri|
+    fiber_id = fiber.tag || fiber.inspect
+    case event
+    when :fiber_schedule
+      log format("=> %s %s %s %s", event, fiber_id, value.inspect, pri ? '(priority)' : '')
+    when :fiber_run
+      log format("=> %s %s %s", event, fiber_id, value.inspect)
+    when :fiber_create, :fiber_terminate
+      log format("=> %s %s", event, fiber_id)
+    else
+      log format("=> %s", event)
+    end
+  end
+end