tipi 0.40 → 0.41

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3e78bf3b20c7ec6704c26869694cf69e389ae1c0c2f2e44c82d0104d2ecaded9
-  data.tar.gz: dac8b0014cbc3a6ee5b146f3645bd38f3d55b21d5c17d969eecaa62c1e8b9306
+  metadata.gz: 5a83c181c05d8d8c8de50f236feab65b800b1bd266c2f0a4d4696d4853ac39fd
+  data.tar.gz: d9a84fc9f72c3a5565f587e35fb4ed206c83c863fbb0b4b470449919a687463b
 SHA512:
-  metadata.gz: fa53cedcd271ba9dbc37821315d27bf14b1eac17451c912eb3dcaa3ceae3b0b738f5d10678cf390f0534edd9d352008e71dec95c46566cad513e3b2d783ea000
-  data.tar.gz: 3cf95c22b05c8eaebdf646963d930977ddb010c3ca9174f081cee674e315f5ac672059b31f833ec337c78e3ec43770f72f6bba145ad00e2bd8c1ed8d0acb2174
+  metadata.gz: e9c13f387c3431888d9d43c88d5bd00831c4e2e8976bf8ae4e1acc479b8d50fa888cd58af0fc433da2dab8e3ea09b99ca0adbdcc61abc71457a805c26aa03687
+  data.tar.gz: df01a54c7c46928c67da321cfc830d8f33e8af7fa8197cd5930f3de73838efd4849d51b10af35b1a5297f7f484b296ff2ed6146dbb8d66476bf0b714f1dbfd68

data/.gitignore

@@ -54,4 +54,6 @@ build-iPhoneSimulator/
 
 # Used by RuboCop. Remote config files pulled in from inherit_from directive.
 # .rubocop-https?--*
-log
+log
+
+lib/tipi_ext*

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 0.41 2021-07-26
+
+- Fix Rack adapter (#11)
+- Introduce experimental HTTP/1 parser
+- More work on DF server
+- Allow setting chunk size in `#respond_from_io`
+
 ## 0.40 2021-06-24
 
 - Implement serving static files using splice_chunks (nice performance boost for

data/Gemfile.lock

@@ -1,13 +1,13 @@
 PATH
   remote: .
   specs:
-    tipi (0.40)
+    tipi (0.41)
       acme-client (~> 2.0.8)
       http-2 (~> 0.10.0)
       http_parser.rb (~> 0.6.0)
       msgpack (~> 1.4.2)
-      polyphony (~> 0.57.0)
-      qeweney (~> 0.10.0)
+      polyphony (~> 0.64)
+      qeweney (~> 0.11)
       rack (>= 2.0.8, < 2.3.0)
       websocket (~> 1.2.8)
 
@@ -18,40 +18,48 @@ GEM
       faraday (>= 0.17, < 2.0.0)
     ansi (1.5.0)
     builder (3.2.4)
-    docile (1.3.2)
+    cuba (3.9.3)
+      rack (>= 1.6.0)
+    docile (1.4.0)
     escape_utils (1.2.1)
-    faraday (1.4.3)
+    faraday (1.5.1)
       faraday-em_http (~> 1.0)
       faraday-em_synchrony (~> 1.0)
       faraday-excon (~> 1.1)
+      faraday-httpclient (~> 1.0.1)
       faraday-net_http (~> 1.0)
       faraday-net_http_persistent (~> 1.1)
+      faraday-patron (~> 1.0)
       multipart-post (>= 1.2, < 3)
       ruby2_keywords (>= 0.0.4)
     faraday-em_http (1.0.0)
     faraday-em_synchrony (1.0.0)
     faraday-excon (1.1.0)
+    faraday-httpclient (1.0.1)
     faraday-net_http (1.0.1)
-    faraday-net_http_persistent (1.1.0)
+    faraday-net_http_persistent (1.2.0)
+    faraday-patron (1.0.0)
     http-2 (0.10.2)
     http_parser.rb (0.6.0)
-    json (2.3.1)
-    localhost (1.1.4)
+    json (2.5.1)
+    localhost (1.1.8)
     minitest (5.11.3)
-    minitest-reporters (1.4.2)
+    minitest-reporters (1.4.3)
       ansi
       builder
       minitest (>= 5.0)
       ruby-progressbar
     msgpack (1.4.2)
     multipart-post (2.1.1)
-    polyphony (0.57.0)
-    qeweney (0.10)
+    polyphony (0.64)
+    qeweney (0.11)
       escape_utils (~> 1.2.1)
     rack (2.2.3)
     rake (12.3.3)
-    ruby-progressbar (1.10.1)
-    ruby2_keywords (0.0.4)
+    rake-compiler (1.1.1)
+      rake
+    ruby-progressbar (1.11.0)
+    ruby2_keywords (0.0.5)
     simplecov (0.17.1)
       docile (~> 1.1)
       json (>= 1.8, < 3)
@@ -63,10 +71,12 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  cuba (~> 3.9.3)
   localhost (~> 1.1.4)
   minitest (~> 5.11.3)
   minitest-reporters (~> 1.4.2)
   rake (~> 12.3.3)
+  rake-compiler (= 1.1.1)
   simplecov (~> 0.17.1)
   tipi!
 

data/Rakefile

@@ -3,10 +3,14 @@
 require "bundler/gem_tasks"
 require "rake/clean"
 
-# frozen_string_literal: true
+require "rake/extensiontask"
+Rake::ExtensionTask.new("tipi_ext") do |ext|
+  ext.ext_dir = "ext/tipi"
+end
+
+task :recompile => [:clean, :compile]
+task :default => [:compile, :test]
 
-task :default => [:test]
 task :test do
   exec 'ruby test/run.rb'
 end
-

data/df/server.rb

@@ -1,111 +1,23 @@
 # frozen_string_literal: true
 
-require 'bundler/setup'
-require 'tipi'
-require 'tipi/digital_fabric'
-require 'tipi/digital_fabric/executive'
-require 'json'
-require 'fileutils'
-require 'localhost/authority'
+require_relative 'server_utils'
 
-FileUtils.cd(__dir__)
-
-service = DigitalFabric::Service.new(token: 'foobar')
-executive = DigitalFabric::Executive.new(service, { host: 'executive.realiteq.net' })
-
-GC.disable
-Thread.current.backend.idle_gc_period = 60
-
-# spin_loop(interval: 60) { GC.start }
-
-class Polyphony::BaseException
-  attr_reader :caller_backtrace
-end
-
-puts "pid: #{Process.pid}"
-
-http_listener = spin do
-  opts = {
-    reuse_addr:  true,
-    dont_linger: true,
-  }
-  puts 'Listening for HTTP on localhost:10080'
-  server = Polyphony::Net.tcp_listen('0.0.0.0', 10080, opts)
-  server.accept_loop do |client|
-    spin do
-      service.incr_connection_count
-      Tipi.client_loop(client, opts) { |req| service.http_request(req) }
-    ensure
-      service.decr_connection_count
-    end
-  rescue Exception => e
-    puts "HTTP accept_loop error: #{e.inspect}"
-    puts e.backtrace.join("\n")
-  end
-end
-
-CERTIFICATE_REGEXP = /(-----BEGIN CERTIFICATE-----\n[^-]+-----END CERTIFICATE-----\n)/.freeze
-
-https_listener = spin do
-  private_key = OpenSSL::PKey::RSA.new IO.read('../../reality/ssl/privkey.pem')
-  c = IO.read('../../reality/ssl/cacert.pem')
-  certificates = c.scan(CERTIFICATE_REGEXP).map { |p|  OpenSSL::X509::Certificate.new(p.first) }
-  ctx = OpenSSL::SSL::SSLContext.new
-  cert = certificates.shift
-  puts "Certificate expires: #{cert.not_after.inspect}"
-  ctx.add_certificate(cert, private_key, certificates)
-  ctx.ciphers = 'ECDH+aRSA'
-
-  # TODO: further limit ciphers
-  # ref: https://github.com/socketry/falcon/blob/3ec805b3ceda0a764a2c5eb68cde33897b6a35ff/lib/falcon/environments/tls.rb
-  # ref: https://github.com/socketry/falcon/blob/3ec805b3ceda0a764a2c5eb68cde33897b6a35ff/lib/falcon/tls.rb
-
-  opts = {
-    reuse_addr:     true,
-    dont_linger:    true,
-    secure_context: ctx,
-    alpn_protocols: Tipi::ALPN_PROTOCOLS
-  }
-
-  puts 'Listening for HTTPS on localhost:10443'
-  server = Polyphony::Net.tcp_listen('0.0.0.0', 10443, opts)
-  loop do
-    client = server.accept
-    spin do
-      service.incr_connection_count
-      Tipi.client_loop(client, opts) { |req| service.http_request(req) }
-    rescue Exception => e
-      puts "Exception: #{e.inspect}"
-      puts e.backtrace.join("\n")
-    ensure
-      service.decr_connection_count
-    end
-  rescue Polyphony::BaseException
-    raise
-  rescue OpenSSL::SSL::SSLError, SystemCallError => e
-    puts "HTTPS accept error: #{e.inspect}"
-  rescue Exception => e
-    puts "HTTPS accept error: #{e.inspect}"
-    puts e.backtrace.join("\n")
-  end
-end
-
-UNIX_SOCKET_PATH = '/tmp/df.sock'
-unix_listener = spin do
-  puts "Listening on #{UNIX_SOCKET_PATH}"
-  FileUtils.rm(UNIX_SOCKET_PATH) if File.exists?(UNIX_SOCKET_PATH)
-  socket = UNIXServer.new(UNIX_SOCKET_PATH)
-  Tipi.accept_loop(socket, {}) { |req| service.http_request(req) }
-end
+listeners = [
+  listen_http,
+  listen_https,
+  listen_unix
+]
 
 begin
-  Fiber.await(http_listener, https_listener, unix_listener)
+  log('Starting DF server')
+  Fiber.await(*listeners)
 rescue Interrupt
-  puts "Got SIGINT, shutting down gracefully"
-  service.graceful_shutdown
-  puts "post graceful shutdown"
+  log('Got SIGINT, shutting down gracefully')
+  @service.graceful_shutdown
+rescue SystemExit
+  # ignore
 rescue Exception => e
-  puts '*' * 40
-  p e
-  puts e.backtrace.join("\n")
+  log("Uncaught exception", error: e, backtrace: e.backtrace)
+ensure
+  log('DF server stopped')
 end

data/df/server_utils.rb

@@ -0,0 +1,173 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'tipi'
+require 'tipi/digital_fabric'
+require 'tipi/digital_fabric/executive'
+require 'json'
+require 'fileutils'
+require 'time'
+
+module ::Kernel
+  def trace(*args)
+    STDOUT.orig_write(format_trace(args))
+  end
+
+  def format_trace(args)
+    if args.first.is_a?(String)
+      if args.size > 1
+        format("%s: %p\n", args.shift, args)
+      else
+        format("%s\n", args.first)
+      end
+    else
+      format("%p\n", args.size == 1 ? args.first : args)
+    end
+  end
+end
+
+FileUtils.cd(__dir__)
+
+@service = DigitalFabric::Service.new(token: 'foobar')
+@executive = DigitalFabric::Executive.new(@service, { host: '@executive.realiteq.net' })
+
+@pid = Process.pid
+
+def log(msg, **ctx)
+  text = format(
+    "%s (%d) %s\n",
+    Time.now.strftime('%Y-%m-%d %H:%M:%S.%3N'),
+    @pid,
+    msg
+  )
+  STDOUT.orig_write text
+  return if ctx.empty?
+
+  ctx.each { |k, v| STDOUT.orig_write format("  %s: %s\n", k, v.inspect) }
+end
+
+def listen_http
+  spin(:http_listener) do
+    opts = {
+      reuse_addr:  true,
+      dont_linger: true,
+    }
+    log('Listening for HTTP on localhost:10080')
+    server = Polyphony::Net.tcp_listen('0.0.0.0', 10080, opts)
+    id = 0
+    loop do
+      client = server.accept
+      # log("Accept HTTP connection", client: client)
+      spin("http#{id += 1}") do
+        @service.incr_connection_count
+        Tipi.client_loop(client, opts) { |req| @service.http_request(req) }
+      ensure
+        # log("Done with HTTP connection", client: client)
+        @service.decr_connection_count
+      end
+    rescue => e
+      log("HTTP accept loop error", error: e, backtrace: e.backtrace)
+    end
+  end
+end
+
+CERTIFICATE_REGEXP = /(-----BEGIN CERTIFICATE-----\n[^-]+-----END CERTIFICATE-----\n)/.freeze
+
+def listen_https
+  spin('https_listener') do
+    private_key = OpenSSL::PKey::RSA.new IO.read('../../reality/ssl/privkey.pem')
+    c = IO.read('../../reality/ssl/cacert.pem')
+    certificates = c.scan(CERTIFICATE_REGEXP).map { |p|  OpenSSL::X509::Certificate.new(p.first) }
+    ctx = OpenSSL::SSL::SSLContext.new
+    cert = certificates.shift
+    log "SSL Certificate expires: #{cert.not_after.inspect}"
+    ctx.add_certificate(cert, private_key, certificates)
+    ctx.ciphers = 'ECDH+aRSA'
+
+    # TODO: further limit ciphers
+    # ref: https://github.com/socketry/falcon/blob/3ec805b3ceda0a764a2c5eb68cde33897b6a35ff/lib/falcon/environments/tls.rb
+    # ref: https://github.com/socketry/falcon/blob/3ec805b3ceda0a764a2c5eb68cde33897b6a35ff/lib/falcon/tls.rb
+
+    opts = {
+      reuse_addr:     true,
+      dont_linger:    true,
+      secure_context: ctx,
+      alpn_protocols: Tipi::ALPN_PROTOCOLS
+    }
+
+    log('Listening for HTTPS on localhost:10443')
+    server = Polyphony::Net.tcp_listen('0.0.0.0', 10443, opts)
+    id = 0
+    loop do
+      client = server.accept
+      # log('Accept HTTPS client connection', client: client)
+      spin("https#{id += 1}") do
+        @service.incr_connection_count
+        Tipi.client_loop(client, opts) { |req| @service.http_request(req) }
+      rescue => e
+        log('Error while handling HTTPS client', client: client, error: e, backtrace: e.backtrace)
+      ensure
+        # log("Done with HTTP connection", client: client)
+        @service.decr_connection_count
+      end
+    rescue OpenSSL::SSL::SSLError, SystemCallError, TypeError => e
+      # log('HTTPS accept error', error: e, backtrace: e.backtrace)
+    rescue => e
+      log('HTTPS accept (unknown) error', error: e, backtrace: e.backtrace)
+    end
+  end
+end
+
+UNIX_SOCKET_PATH = '/tmp/df.sock'
+def listen_unix
+  spin(:unix_listener) do
+    log("Listening on #{UNIX_SOCKET_PATH}")
+    FileUtils.rm(UNIX_SOCKET_PATH) if File.exists?(UNIX_SOCKET_PATH)
+    socket = UNIXServer.new(UNIX_SOCKET_PATH)
+
+    id = 0
+    socket.accept_loop do |client|
+      # log('Accept Unix connection', client: client)
+      spin("unix#{id += 1}") do
+        Tipi.client_loop(client, {}) { |req| @service.http_request(req, true) }
+      end
+    rescue OpenSSL::SSL::SSLError
+      # disregard
+    end
+  end
+end
+
+def listen_df
+  spin(:df_listener) do
+    opts = {
+      reuse_addr:  true,
+      dont_linger: true,
+    }
+    log('Listening for DF connections on localhost:4321')
+    server = Polyphony::Net.tcp_listen('0.0.0.0', 4321, opts)
+
+    id = 0
+    server.accept_loop do |client|
+      # log('Accept DF connection', client: client)
+      spin("df#{id += 1}") do
+        Tipi.client_loop(client, {}) { |req| @service.http_request(req, true) }
+      end
+    rescue OpenSSL::SSL::SSLError
+      # disregard
+    end
+  end
+end
+
+# Thread.backend.trace_proc = proc do |event, fiber, value, pri|
+#   fiber_id = fiber.tag || fiber.inspect
+#   case event
+#   when :fiber_schedule
+#     log format("=> %s %s %s %s", event, fiber_id, value.inspect, pri ? '' : '(priority)')
+#   when :fiber_run
+#     log format("=> %s %s %s", event, fiber_id, value.inspect)
+#   when :fiber_create, :fiber_terminate
+#     log format("=> %s %s", event, fiber_id)
+#   else
+#     log format("=> %s", event)
+#   end
+# end

data/examples/http1_parser.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require 'polyphony'
+require_relative '../lib/tipi_ext'
+
+i, o = IO.pipe
+
+module ::Kernel
+  def trace(*args)
+    STDOUT.orig_write(format_trace(args))
+  end
+
+  def format_trace(args)
+    if args.first.is_a?(String)
+      if args.size > 1
+        format("%s: %p\n", args.shift, args)
+      else
+        format("%s\n", args.first)
+      end
+    else
+      format("%p\n", args.size == 1 ? args.first : args)
+    end
+  end
+end
+
+f = spin do
+  parser = Tipi::HTTP1Parser.new(i)
+  while true
+    trace '*' * 40
+    headers = parser.parse_headers
+    break unless headers
+    trace headers
+
+    body = parser.read_body(headers)
+    trace "body: #{body ? body.bytesize : 0} bytes"
+  end
+end
+
+o << "GET /a HTTP/1.1\r\n\r\n"
+o << "GET /b HTTP/1.1\r\n\r\n"
+
+# o << "GET / HTTP/1.1\r\nHost: localhost:10080\r\nUser-Agent: curl/7.74.0\r\nAccept: */*\r\n\r\n"
+
+# o << "post /?q=time&blah=blah HTTP/1\r\nHost: dev.realiteq.net\r\n\r\n"
+
+# data = " " * 4000000
+# o << "get /?q=time HTTP/1.1\r\nContent-Length: #{data.bytesize}\r\n\r\n#{data}"
+
+# o << "get /?q=time HTTP/1.1\r\nCookie: foo\r\nCookie: bar\r\n\r\n"
+
+# o.close
+ 
+f.await