tipi 0.39 → 0.43

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 02c5ce4798e1961ca59798b75bafe158570e777661f6fc8667169553df3cd3f4
-  data.tar.gz: 405bfa3526efaad394d34840764dc131230a366574cfc52b63344a6d5cdfe6dd
+  metadata.gz: c7a42c93f436a24b74aef2fc9de54d7f01b587ec42b2897d0b0c24a9090776d8
+  data.tar.gz: 1fa532504e9c79d620306b4ddfbd01bf1d1dc39b563ec2a2c3cfec9cb00a1b64
 SHA512:
-  metadata.gz: 754bd35136e4e004e6bd782eb2060d933b2439c437a9c2a966529e0465c42096b97fa08b1dae04b7a3c8aa0c77c367fd5395d215cca73fd7cf1aa80c297b3178
-  data.tar.gz: 24425d798c076ad43b4fa45b641a90a5e801dfe35acd6217e8a8fc97ea546f18457fc9a8653e47f7bd5f696da885ef86db8db36385dcaa79ccbcf9d42d64081c
+  metadata.gz: 93282cde73eca46289cf212e76c90e7390dcd9e48e8594bd756d5f59ae3bc5fed41e61d216c85cca9c9d2c8fca1061ee0159f8cc0ca575fe0b7149a21b1edc24
+  data.tar.gz: 6fd1dc4de05351d0b7da91a7344798145f31659f4663f451355222ce63bb8504fb123c562199820b4f217bc14d064bb529fcc0c9ea495863856bd79f073a491b

data/.github/workflows/test.yml

@@ -23,5 +23,9 @@ jobs:
       run: |
         gem install bundler
         POLYPHONY_USE_LIBEV=1 bundle install
+    - name: Show Linux kernel version
+      run: uname -r
+    - name: Compile C-extension
+      run: bundle exec rake compile
     - name: Run tests
       run:  bundle exec rake test

data/.gitignore

@@ -54,4 +54,8 @@ build-iPhoneSimulator/
 
 # Used by RuboCop. Remote config files pulled in from inherit_from directive.
 # .rubocop-https?--*
-log
+log
+log.*
+
+lib/tipi_ext*
+examples/certificate_store.db

data/CHANGELOG.md

@@ -1,3 +1,33 @@
+## 0.43 2021-08-20
+
+- Extract HTTP/1 parser into a separate gem:
+  [H1P](https://github.com/digital-fabric/h1p)
+
+## 0.42 2021-08-16
+
+- HTTP/1 parser:  disable UTF-8 parsing for all but header values
+- Add support for parsing HTTP/1 from callable source
+- Introduce full_service API for automatic HTTPS
+- Introduce automatic SSL certificate provisioning
+- Improve handling of exceptions
+- Various fixes to DF service and agent pxoy
+- Fix upgrading to HTTP2 with a request body
+- Switch to new HTTP/1 parser
+
+## 0.41 2021-07-26
+
+- Fix Rack adapter (#11)
+- Introduce experimental HTTP/1 parser
+- More work on DF server
+- Allow setting chunk size in `#respond_from_io`
+
+## 0.40 2021-06-24
+
+- Implement serving static files using splice_chunks (nice performance boost for
+  files bigger than 1M)
+- Call shutdown before closing socket
+- Fix examples (thanks @timhatch!)
+
 ## 0.39 2021-06-20
 
 - More work on DF server

data/Gemfile

@@ -1,3 +1,7 @@
 source 'https://rubygems.org'
 
-gemspec
+gemspec
+%w{polyphony qeweney h1p}.each do |dep|
+  dir = "../#{dep}"
+  gem(dep, path: dir) if File.directory?(dir)
+end

data/Gemfile.lock

@@ -1,39 +1,73 @@
+PATH
+  remote: ../h1p
+  specs:
+    h1p (0.2)
+
+PATH
+  remote: ../polyphony
+  specs:
+    polyphony (0.70)
+
+PATH
+  remote: ../qeweney
+  specs:
+    qeweney (0.14)
+      escape_utils (~> 1.2.1)
+
 PATH
   remote: .
   specs:
-    tipi (0.39)
-      http-2 (~> 0.10.0)
-      http_parser.rb (~> 0.6.0)
+    tipi (0.43)
+      acme-client (~> 2.0.8)
+      extralite (~> 1.2)
+      h1p (~> 0.2)
+      http-2 (~> 0.11)
+      localhost (~> 1.1.4)
       msgpack (~> 1.4.2)
-      polyphony (~> 0.55.0)
-      qeweney (~> 0.9.1)
+      polyphony (~> 0.69)
+      qeweney (~> 0.14)
       rack (>= 2.0.8, < 2.3.0)
       websocket (~> 1.2.8)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    ansi (1.5.0)
-    builder (3.2.4)
-    docile (1.3.2)
+    acme-client (2.0.8)
+      faraday (>= 0.17, < 2.0.0)
+    cuba (3.9.3)
+      rack (>= 1.6.0)
+    docile (1.4.0)
     escape_utils (1.2.1)
-    http-2 (0.10.2)
-    http_parser.rb (0.6.0)
-    json (2.3.1)
-    localhost (1.1.4)
+    extralite (1.3)
+    faraday (1.7.0)
+      faraday-em_http (~> 1.0)
+      faraday-em_synchrony (~> 1.0)
+      faraday-excon (~> 1.1)
+      faraday-httpclient (~> 1.0.1)
+      faraday-net_http (~> 1.0)
+      faraday-net_http_persistent (~> 1.1)
+      faraday-patron (~> 1.0)
+      faraday-rack (~> 1.0)
+      multipart-post (>= 1.2, < 3)
+      ruby2_keywords (>= 0.0.4)
+    faraday-em_http (1.0.0)
+    faraday-em_synchrony (1.0.0)
+    faraday-excon (1.1.0)
+    faraday-httpclient (1.0.1)
+    faraday-net_http (1.0.1)
+    faraday-net_http_persistent (1.2.0)
+    faraday-patron (1.0.0)
+    faraday-rack (1.0.0)
+    http-2 (0.11.0)
+    json (2.5.1)
+    localhost (1.1.8)
+    memory_profiler (1.0.0)
     minitest (5.11.3)
-    minitest-reporters (1.4.2)
-      ansi
-      builder
-      minitest (>= 5.0)
-      ruby-progressbar
     msgpack (1.4.2)
-    polyphony (0.55.0)
-    qeweney (0.9.1)
-      escape_utils (~> 1.2.1)
+    multipart-post (2.1.1)
     rack (2.2.3)
-    rake (12.3.3)
-    ruby-progressbar (1.10.1)
+    rake (13.0.6)
+    ruby2_keywords (0.0.5)
     simplecov (0.17.1)
       docile (~> 1.1)
       json (>= 1.8, < 3)
@@ -45,10 +79,13 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
-  localhost (~> 1.1.4)
+  cuba (~> 3.9.3)
+  h1p!
+  memory_profiler (~> 1.0.0)
   minitest (~> 5.11.3)
-  minitest-reporters (~> 1.4.2)
-  rake (~> 12.3.3)
+  polyphony!
+  qeweney!
+  rake (~> 13.0.6)
   simplecov (~> 0.17.1)
   tipi!
 

data/Rakefile

@@ -3,10 +3,14 @@
 require "bundler/gem_tasks"
 require "rake/clean"
 
-# frozen_string_literal: true
+require "rake/extensiontask"
+Rake::ExtensionTask.new("tipi_ext") do |ext|
+  ext.ext_dir = "ext/tipi"
+end
+
+task :recompile => [:clean, :compile]
+task :default => [:compile, :test]
 
-task :default => [:test]
 task :test do
   exec 'ruby test/run.rb'
 end
-

data/benchmarks/bm_http1_parser.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+
+HTTP_REQUEST = "GET /foo HTTP/1.1\r\nHost: example.com\r\nAccept: */*\r\n\r\n"
+
+def measure_time_and_allocs
+  4.times { GC.start }
+  GC.disable
+
+  t0 = Time.now
+  a0 = object_count
+  yield
+  t1 = Time.now
+  a1 = object_count
+  [t1 - t0, a1 - a0]
+ensure
+  GC.enable
+end
+
+def object_count
+  count = ObjectSpace.count_objects
+  count[:TOTAL] - count[:FREE]
+end
+
+def benchmark_other_http1_parser(iterations)
+  STDOUT << "http_parser.rb: "
+  require 'http_parser.rb'
+  
+  i, o = IO.pipe
+  parser = Http::Parser.new
+  done = false
+  headers = nil
+  parser.on_headers_complete = proc do |h|
+    headers = h
+    headers[':method'] = parser.http_method
+    headers[':path'] = parser.request_url
+  end
+  parser.on_message_complete = proc { done = true }
+
+  elapsed, allocated = measure_time_and_allocs do
+    iterations.times do
+      o << HTTP_REQUEST
+      done = false
+      while !done
+        msg = i.readpartial(4096)
+        parser << msg
+      end
+    end
+  end
+  puts(format('elapsed: %f, allocated: %d (%f/req), rate: %f ips', elapsed, allocated, allocated.to_f / iterations, iterations / elapsed))
+end
+
+def benchmark_tipi_http1_parser(iterations)
+  STDOUT << "tipi parser: "
+  require_relative '../lib/tipi_ext'
+  i, o = IO.pipe
+  reader = proc { |len| i.readpartial(len) }
+  parser = Tipi::HTTP1Parser.new(reader)
+
+  elapsed, allocated = measure_time_and_allocs do
+    iterations.times do
+      o << HTTP_REQUEST
+      headers = parser.parse_headers
+    end
+  end
+  puts(format('elapsed: %f, allocated: %d (%f/req), rate: %f ips', elapsed, allocated, allocated.to_f / iterations, iterations / elapsed))
+end
+
+def fork_benchmark(method, iterations)
+  pid = fork do
+    send(method, iterations)
+  rescue Exception => e
+    p e
+    p e.backtrace
+    exit!
+  end
+  Process.wait(pid)
+end
+
+x = 500000
+# fork_benchmark(:benchmark_other_http1_parser, x)
+# fork_benchmark(:benchmark_tipi_http1_parser, x)
+
+benchmark_tipi_http1_parser(x)

data/bin/benchmark

@@ -0,0 +1,37 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'polyphony'
+
+def parse_latency(latency)
+  m = latency.match(/^([\d\.]+)(us|ms|s)$/)
+  return nil unless m
+
+  value = m[1].to_f
+  case m[2]
+  when 's' then value
+  when 'ms' then value / 1000
+  when 'us' then value / 1000000
+  end
+end
+
+def parse_wrk_results(results)
+  lines = results.lines
+  latencies = lines[3].strip.split(/\s+/)
+  throughput = lines[6].strip.split(/\s+/)
+
+  {
+    latency_avg:  parse_latency(latencies[1]),
+    latency_max:  parse_latency(latencies[3]),
+    rate:         throughput[1].to_f
+  }
+end
+
+def run_wrk(duration: 10, threads: 2, connections: 10, url: )
+  `wrk -d#{duration} -t#{threads} -c#{connections} #{url}`
+end
+
+[8, 64, 256, 512].each do |c|
+  puts "connections: #{c}"
+  p parse_wrk_results(run_wrk(duration: 10, threads: 4, connections: c, url: "http://localhost:10080/"))
+end

data/bin/h1pd

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -e
+rake compile
+ruby test/test_http1_parser.rb
+ruby benchmarks/bm_http1_parser.rb

data/bin/tipi

@@ -1,26 +1,8 @@
 #!/usr/bin/env ruby
 
 require 'bundler/setup'
-require 'polyphony'
-require File.expand_path('../lib/tipi/configuration', __dir__)
+require 'tipi/cli'
 
-config = {}
-#config[:forked] = 4
+trap('SIGINT') { exit }
 
-puts DATA.read
-puts
-
-configuration_manager = spin { Tipi::Configuration.supervise_config }
-
-configuration_manager << config
-configuration_manager.await
-
-__END__
-
-          ooo
-       oo
-     o
-   \|/
-   / \   Tipi - A better web server for a better world
-  /___\
-  
+Tipi::CLI.start

data/df/server.rb

@@ -1,96 +1,25 @@
 # frozen_string_literal: true
 
-require 'bundler/setup'
-require 'tipi'
-require 'tipi/digital_fabric'
-require 'tipi/digital_fabric/executive'
-require 'json'
-require 'fileutils'
-require 'localhost/authority'
+require_relative 'server_utils'
 
-FileUtils.cd(__dir__)
+listeners = [
+  listen_http,
+  listen_https,
+  listen_unix
+]
 
-service = DigitalFabric::Service.new(token: 'foobar')
-executive = DigitalFabric::Executive.new(service, { host: 'executive.realiteq.net' })
-
-spin_loop(interval: 60) { GC.start }
-
-class Polyphony::BaseException
-  attr_reader :caller_backtrace
-end
-
-puts "pid: #{Process.pid}"
-
-http_listener = spin do
-  opts = {
-    reuse_addr:  true,
-    dont_linger: true,
-  }
-  puts 'Listening for HTTP on localhost:10080'
-  server = Polyphony::Net.tcp_listen('0.0.0.0', 10080, opts)
-  server.accept_loop do |client|
-    spin do
-      service.incr_connection_count
-      Tipi.client_loop(client, opts) { |req| service.http_request(req) }
-    ensure
-      service.decr_connection_count
-    end
-  rescue Exception => e
-    puts "HTTP accept_loop error: #{e.inspect}"
-    puts e.backtrace.join("\n")
-  end
-end
-
-CERTIFICATE_REGEXP = /(-----BEGIN CERTIFICATE-----\n[^-]+-----END CERTIFICATE-----\n)/.freeze
-
-https_listener = spin do
-  private_key = OpenSSL::PKey::RSA.new IO.read('../../reality/ssl/privkey.pem')
-  c = IO.read('../../reality/ssl/cacert.pem')
-  certificates = c.scan(CERTIFICATE_REGEXP).map { |p|  OpenSSL::X509::Certificate.new(p.first) }
-  ctx = OpenSSL::SSL::SSLContext.new
-  cert = certificates.shift
-  puts "Certificate expires: #{cert.not_after.inspect}"
-  ctx.add_certificate(cert, private_key, certificates)
-  # ctx = Localhost::Authority.fetch.server_context
-  opts = {
-    reuse_addr:     true,
-    dont_linger:    true,
-    secure_context: ctx,
-    alpn_protocols: Tipi::ALPN_PROTOCOLS
-  }
-
-  puts 'Listening for HTTPS on localhost:10443'
-  server = Polyphony::Net.tcp_listen('0.0.0.0', 10443, opts)
-  server.accept_loop do |client|
-    spin do
-      service.incr_connection_count
-      Tipi.client_loop(client, opts) { |req| service.http_request(req) }
-    ensure
-      service.decr_connection_count
-    end
-  rescue Exception => e
-    puts "HTTPS accept_loop error: #{e.inspect}"
-    puts e.backtrace.join("\n")
-  end
-end
-
-UNIX_SOCKET_PATH = '/tmp/df.sock'
-
-unix_listener = spin do
-  puts "Listening on #{UNIX_SOCKET_PATH}"
-  FileUtils.rm(UNIX_SOCKET_PATH) if File.exists?(UNIX_SOCKET_PATH)
-  socket = UNIXServer.new(UNIX_SOCKET_PATH)
-  Tipi.accept_loop(socket, {}) { |req| service.http_request(req) }
-end
+spin_loop(interval: 60) { GC.compact } if GC.respond_to?(:compact)
 
 begin
-  Fiber.await(http_listener, https_listener, unix_listener)
+  log('Starting DF server')
+  Fiber.await(*listeners)
 rescue Interrupt
-  puts "Got SIGINT, shutting down gracefully"
-  service.graceful_shutdown
-  puts "post graceful shutdown"
+  log('Got SIGINT, shutting down gracefully')
+  @service.graceful_shutdown
+rescue SystemExit
+  # ignore
 rescue Exception => e
-  puts '*' * 40
-  p e
-  puts e.backtrace.join("\n")
+  log("Uncaught exception", error: e, source: e.source_fiber, raising: e.raising_fiber, backtrace: e.backtrace)
+ensure
+  log('DF server stopped')
 end