tipi 0.38 → 0.42

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6a8c075ce0b769f014a20587dd061f17e32361f211aad5741f655dafe6687e37
-  data.tar.gz: d14c0fe026a7db1aa46edded54e5ee6f1a0f8598f626dd1aff342ad07b5cec39
+  metadata.gz: 82155f4b86223d3fb32dc8c314abfeb75fda951747e63832ed5c15bc3b1f43cc
+  data.tar.gz: 721b98fb8d330d1bc38f8f236f82ddd909235d31f5875c1b4f89895e4e3926da
 SHA512:
-  metadata.gz: e306783fc83d9d7ebd0678c4d68a13f6f9bd77e4b9d83fc84d745e5cc3f53eb27b69286fb1ba9f1ca54fe57358009a4c11dc9374c8a8d9046dc4330e359dc988
-  data.tar.gz: 789c3b4e1374cc57e04e3bce4aa8d12022cfd7e31c9cba4f5777302f57943da0a967f17f8cb2f91551357deac1bbd07fe514c2cb2e002331f525d341c6b7f0e9
+  metadata.gz: 10389779975234d90c968626b4f9d6b168491f9a32ecf1be71683c37cf11f58f1bac52a36478691e72328236a16d309a8462104bda556772e25cebe52b2b0d53
+  data.tar.gz: 99db2c15278cdfc286ffad05f266f5d62eec6a3bb1363f8dfa4c302549a308a691496ee60ec085ede59e3b594533e8e196bbca144878e5a6dccf2ecfc962151c

data/.github/workflows/test.yml

@@ -22,6 +22,10 @@ jobs:
     - name: Install dependencies
       run: |
         gem install bundler
-        bundle install
+        POLYPHONY_USE_LIBEV=1 bundle install
+    - name: Show Linux kernel version
+      run: uname -r
+    - name: Compile C-extension
+      run: bundle exec rake compile
     - name: Run tests
       run:  bundle exec rake test

data/.gitignore

@@ -54,3 +54,8 @@ build-iPhoneSimulator/
 
 # Used by RuboCop. Remote config files pulled in from inherit_from directive.
 # .rubocop-https?--*
+log
+log.*
+
+lib/tipi_ext*
+examples/certificate_store.db

data/CHANGELOG.md

@@ -1,3 +1,37 @@
+## 0.42 2021-08-16
+
+- HTTP/1 parser:  disable UTF-8 parsing for all but header values
+- Add support for parsing HTTP/1 from callable source
+- Introduce full_service API for automatic HTTPS
+- Introduce automatic SSL certificate provisioning
+- Improve handling of exceptions
+- Various fixes to DF service and agent pxoy
+- Fix upgrading to HTTP2 with a request body
+- Switch to new HTTP/1 parser
+
+## 0.41 2021-07-26
+
+- Fix Rack adapter (#11)
+- Introduce experimental HTTP/1 parser
+- More work on DF server
+- Allow setting chunk size in `#respond_from_io`
+
+## 0.40 2021-06-24
+
+- Implement serving static files using splice_chunks (nice performance boost for
+  files bigger than 1M)
+- Call shutdown before closing socket
+- Fix examples (thanks @timhatch!)
+
+## 0.39 2021-06-20
+
+- More work on DF server
+- Fix HTTP2StreamHandler#send_headers
+- Various fixes to HTTP/2 adapter
+- Fix host detection for HTTP/2 connections
+- Fix HTTP/1 adapter #respond with nil body
+- Fix HTTP1Adapter#send_headers
+
 ## 0.38 2021-03-09
 
 - Don't use chunked transfer encoding for non-streaming responses

data/Gemfile

@@ -1,3 +1,7 @@
 source 'https://rubygems.org'
 
-gemspec
+gemspec
+%w{polyphony qeweney}.each do |dep|
+  dir = "../#{dep}"
+  gem(dep, path: dir) if File.directory?(dir)
+end

data/Gemfile.lock

@@ -1,39 +1,77 @@
+PATH
+  remote: ../polyphony
+  specs:
+    polyphony (0.69)
+
+PATH
+  remote: ../qeweney
+  specs:
+    qeweney (0.14)
+      escape_utils (~> 1.2.1)
+
 PATH
   remote: .
   specs:
-    tipi (0.38)
-      http-2 (~> 0.10.0)
-      http_parser.rb (~> 0.6.0)
+    tipi (0.42)
+      acme-client (~> 2.0.8)
+      extralite (~> 1.2)
+      http-2 (~> 0.11)
+      localhost (~> 1.1.4)
       msgpack (~> 1.4.2)
-      polyphony (~> 0.52.0)
-      qeweney (~> 0.6)
+      polyphony (~> 0.69)
+      qeweney (~> 0.14)
       rack (>= 2.0.8, < 2.3.0)
       websocket (~> 1.2.8)
 
 GEM
   remote: https://rubygems.org/
   specs:
+    acme-client (2.0.8)
+      faraday (>= 0.17, < 2.0.0)
     ansi (1.5.0)
     builder (3.2.4)
-    docile (1.3.2)
+    cuba (3.9.3)
+      rack (>= 1.6.0)
+    docile (1.4.0)
     escape_utils (1.2.1)
-    http-2 (0.10.2)
-    http_parser.rb (0.6.0)
-    json (2.3.1)
-    localhost (1.1.4)
+    extralite (1.2)
+    faraday (1.7.0)
+      faraday-em_http (~> 1.0)
+      faraday-em_synchrony (~> 1.0)
+      faraday-excon (~> 1.1)
+      faraday-httpclient (~> 1.0.1)
+      faraday-net_http (~> 1.0)
+      faraday-net_http_persistent (~> 1.1)
+      faraday-patron (~> 1.0)
+      faraday-rack (~> 1.0)
+      multipart-post (>= 1.2, < 3)
+      ruby2_keywords (>= 0.0.4)
+    faraday-em_http (1.0.0)
+    faraday-em_synchrony (1.0.0)
+    faraday-excon (1.1.0)
+    faraday-httpclient (1.0.1)
+    faraday-net_http (1.0.1)
+    faraday-net_http_persistent (1.2.0)
+    faraday-patron (1.0.0)
+    faraday-rack (1.0.0)
+    http-2 (0.11.0)
+    http_parser.rb (0.7.0)
+    json (2.5.1)
+    localhost (1.1.8)
     minitest (5.11.3)
-    minitest-reporters (1.4.2)
+    minitest-reporters (1.4.3)
       ansi
       builder
       minitest (>= 5.0)
       ruby-progressbar
     msgpack (1.4.2)
-    polyphony (0.52.0)
-    qeweney (0.7.5)
-      escape_utils (~> 1.2.1)
+    multipart-post (2.1.1)
     rack (2.2.3)
     rake (12.3.3)
-    ruby-progressbar (1.10.1)
+    rake-compiler (1.1.1)
+      rake
+    ruby-progressbar (1.11.0)
+    ruby2_keywords (0.0.5)
     simplecov (0.17.1)
       docile (~> 1.1)
       json (>= 1.8, < 3)
@@ -45,10 +83,14 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
-  localhost (~> 1.1.4)
+  cuba (~> 3.9.3)
+  http_parser.rb (= 0.7.0)
   minitest (~> 5.11.3)
   minitest-reporters (~> 1.4.2)
+  polyphony!
+  qeweney!
   rake (~> 12.3.3)
+  rake-compiler (= 1.1.1)
   simplecov (~> 0.17.1)
   tipi!
 

data/Rakefile

@@ -3,10 +3,14 @@
 require "bundler/gem_tasks"
 require "rake/clean"
 
-# frozen_string_literal: true
+require "rake/extensiontask"
+Rake::ExtensionTask.new("tipi_ext") do |ext|
+  ext.ext_dir = "ext/tipi"
+end
+
+task :recompile => [:clean, :compile]
+task :default => [:compile, :test]
 
-task :default => [:test]
 task :test do
   exec 'ruby test/run.rb'
 end
-

data/TODO.md

@@ -1,4 +1,80 @@
-For immediate execution:
+## Add an API for reading a request body chunk into an IO (pipe)
+
+      ```ruby
+      # currently
+      chunk = req.next_chunk
+      # or
+      req.each_chunk { |c| do_something(c) }
+
+      # what we'd like to do
+      r, w = IO.pipe
+      len = req.splice_chunk(w)
+      sock << "Here comes a chunk of #{len} bytes\n"
+      sock.splice(r, len)
+
+      # or:
+      r, w = IO.pipe
+      req.splice_each_chunk(w) do |len|
+        sock << "Here comes a chunk of #{len} bytes\n"
+        sock.splice(r, len)
+      end
+      ```
+
+# HTTP/1.1 parser
+
+- httparser.rb is not actively updated
+- the httparser.rb C parser code comes originally from https://github.com/nodejs/llhttp
+- there's a Ruby gem https://github.com/metabahn/llhttp, but its API is too low-level
+  (lots of callbacks, headers need to be retained across callbacks)
+- the basic idea is to import the C-code, then build a parser object with the following
+  callbacks:
+
+  ```ruby
+  on_headers_complete(headers)
+  on_body_chunk(chunk)
+  on_message_complete
+  ```
+
+- The llhttp gem's C-code is here: https://github.com/metabahn/llhttp/tree/main/mri
+
+- Actually, if you do a C extension, instead of a callback-based API, we can
+  design a blocking API:
+
+  ```ruby
+  parser = Tipi::HTTP1::Parser.new
+  parser.each_request(socket) do |headers|
+    request = Request.new(normalize_headers(headers))
+    handle_request(request)
+  end
+  ```
+
+# What about HTTP/2?
+
+It would be a nice exercise in converting a callback-based API to a blocking
+one:
+
+```ruby
+parser = Tipi::HTTP2::Parser.new(socket)
+parser.each_stream(socket) do |stream|
+  spin { handle_stream(stream) }
+end
+```
+
+
+
+# DF
+
+- Add attack protection for IP-address HTTP host:
+
+  ```ruby
+  IPV4_REGEXP = /^\d+\.\d+\.\d+\.\d+$/.freeze
+
+  def is_attack_request?(req)
+    return true if req.host =~ IPV4_REGEXP && req.query[:q] != 'ping'
+  end
+  ```
+
+- Add attack route to Qeweney routing API
 
 
 

data/benchmarks/bm_http1_parser.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+
+HTTP_REQUEST = "GET /foo HTTP/1.1\r\nHost: example.com\r\nAccept: */*\r\n\r\n"
+
+def benchmark_other_http1_parser(iterations)
+  STDOUT << "http_parser.rb: "
+  require 'http_parser.rb'
+  
+  i, o = IO.pipe
+  parser = Http::Parser.new
+  done = false
+  headers = nil
+  parser.on_headers_complete = proc do |h|
+    headers = h
+    headers[':method'] = parser.http_method
+    headers[':path'] = parser.request_url
+    headers[':protocol'] = parser.http_version
+  end
+  parser.on_message_complete = proc { done = true }
+
+  t0 = Time.now
+  iterations.times do
+    o << HTTP_REQUEST
+    done = false
+    while !done
+      msg = i.readpartial(4096)
+      parser << msg
+    end
+  end
+  t1 = Time.now
+  puts "#{iterations / (t1 - t0)} ips"
+end
+
+def benchmark_tipi_http1_parser(iterations)
+  STDOUT << "tipi parser: "
+  require_relative '../lib/tipi_ext'
+  i, o = IO.pipe
+  reader = proc { |len| i.readpartial(len) }
+  parser = Tipi::HTTP1Parser.new(reader)
+
+  t0 = Time.now
+  iterations.times do
+    o << HTTP_REQUEST
+    headers = parser.parse_headers
+  end
+  t1 = Time.now
+  puts "#{iterations / (t1 - t0)} ips"
+end
+
+def fork_benchmark(method, iterations)
+  pid = fork { send(method, iterations) }
+  Process.wait(pid)
+end
+
+x = 500000
+fork_benchmark(:benchmark_other_http1_parser, x)
+fork_benchmark(:benchmark_tipi_http1_parser, x)
+
+# benchmark_tipi_http1_parser(x)

data/bin/benchmark

@@ -0,0 +1,37 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'polyphony'
+
+def parse_latency(latency)
+  m = latency.match(/^([\d\.]+)(us|ms|s)$/)
+  return nil unless m
+
+  value = m[1].to_f
+  case m[2]
+  when 's' then value
+  when 'ms' then value / 1000
+  when 'us' then value / 1000000
+  end
+end
+
+def parse_wrk_results(results)
+  lines = results.lines
+  latencies = lines[3].strip.split(/\s+/)
+  throughput = lines[6].strip.split(/\s+/)
+
+  {
+    latency_avg:  parse_latency(latencies[1]),
+    latency_max:  parse_latency(latencies[3]),
+    rate:         throughput[1].to_f
+  }
+end
+
+def run_wrk(duration: 10, threads: 2, connections: 10, url: )
+  `wrk -d#{duration} -t#{threads} -c#{connections} #{url}`
+end
+
+[8, 64, 256, 512].each do |c|
+  puts "connections: #{c}"
+  p parse_wrk_results(run_wrk(duration: 10, threads: 4, connections: c, url: "http://localhost:10080/"))
+end

data/bin/h1pd

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -e
+rake compile
+ruby test/test_http1_parser.rb
+ruby benchmarks/bm_http1_parser.rb

data/bin/tipi

@@ -1,26 +1,8 @@
 #!/usr/bin/env ruby
 
 require 'bundler/setup'
-require 'polyphony'
-require File.expand_path('../lib/tipi/configuration', __dir__)
+require 'tipi/cli'
 
-config = {}
-#config[:forked] = 4
+trap('SIGINT') { exit }
 
-puts DATA.read
-puts
-
-configuration_manager = spin { Tipi::Configuration.supervise_config }
-
-configuration_manager << config
-configuration_manager.await
-
-__END__
-
-          ooo
-       oo
-     o
-   \|/
-   / \   Tipi - A better web server for a better world
-  /___\
-  
+Tipi::CLI.start

data/df/sample_agent.rb

@@ -13,7 +13,7 @@ class SampleAgent < DigitalFabric::Agent
   HTML_SSE = IO.read(File.join(__dir__, 'sse_page.html'))
 
   def http_request(req)
-    path = req['headers'][':path']
+    path = req.headers[':path']
     case path
     when '/agent'
       send_df_message(Protocol.http_response(

data/df/server.rb

@@ -1,54 +1,23 @@
 # frozen_string_literal: true
 
-require 'bundler/setup'
-require 'tipi'
-require 'tipi/digital_fabric'
-require 'tipi/digital_fabric/executive'
-require 'json'
-require 'fileutils'
-FileUtils.cd(__dir__)
+require_relative 'server_utils'
 
-service = DigitalFabric::Service.new(token: 'foobar')
-executive = DigitalFabric::Executive.new(service, { host: 'executive.realiteq.net' })
-
-spin_loop(interval: 60) { GC.start }
-
-class Polyphony::BaseException
-  attr_reader :caller_backtrace
-end
-
-puts "pid: #{Process.pid}"
-
-tcp_listener = spin do
-  opts = {
-    reuse_addr:  true,
-    dont_linger: true,
-  }
-  puts 'Listening on localhost:4411'
-  server = Polyphony::Net.tcp_listen('0.0.0.0', 4411, opts)
-  server.accept_loop do |client|
-    spin do
-      service.incr_connection_count
-      Tipi.client_loop(client, opts) { |req| service.http_request(req) }
-    ensure
-      service.decr_connection_count
-    end
-  end
-end
-
-UNIX_SOCKET_PATH = '/tmp/df.sock'
-
-unix_listener = spin do
-  puts "Listening on #{UNIX_SOCKET_PATH}"
-  FileUtils.rm(UNIX_SOCKET_PATH) if File.exists?(UNIX_SOCKET_PATH)
-  socket = UNIXServer.new(UNIX_SOCKET_PATH)
-  Tipi.accept_loop(socket, {}) { |req| service.http_request(req) }
-end
+listeners = [
+  listen_http,
+  listen_https,
+  listen_unix
+]
 
 begin
-  Fiber.await(tcp_listener, unix_listener)
+  log('Starting DF server')
+  Fiber.await(*listeners)
 rescue Interrupt
-  puts "Got SIGINT, shutting down gracefully"
-  service.graceful_shutdown
-  puts "post graceful shutdown"
+  log('Got SIGINT, shutting down gracefully')
+  @service.graceful_shutdown
+rescue SystemExit
+  # ignore
+rescue Exception => e
+  log("Uncaught exception", error: e, source: e.source_fiber, raising: e.raising_fiber, backtrace: e.backtrace)
+ensure
+  log('DF server stopped')
 end