tipi 0.37 → 0.40

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4129b10a7f6b5fb92e4e4784bf8cfd12ea2659840b616b7627bf28d1dc423e87
-  data.tar.gz: 7f5f71c4a870e33c7de84fd292568b3aca582dcc1adc9496d99e636328b3df4b
+  metadata.gz: 3e78bf3b20c7ec6704c26869694cf69e389ae1c0c2f2e44c82d0104d2ecaded9
+  data.tar.gz: dac8b0014cbc3a6ee5b146f3645bd38f3d55b21d5c17d969eecaa62c1e8b9306
 SHA512:
-  metadata.gz: c1bfcbf1cdf4fa2868554e690a71543d89a37bf2d8cfa27cf604e453fb411d5b1dd2e4bc10678cbe79db6a94238ffef1a09e53771f7e8d8e71f2512b87f7265a
-  data.tar.gz: 4f476a65800ffb6cbcdfde6e10ed9e1dd7b9fcc4ddd69ad2babed1a747cba4f94e05659cbd235c1f7e71e1595851d77896262d45d1036d216d866b7bdc3f914a
+  metadata.gz: fa53cedcd271ba9dbc37821315d27bf14b1eac17451c912eb3dcaa3ceae3b0b738f5d10678cf390f0534edd9d352008e71dec95c46566cad513e3b2d783ea000
+  data.tar.gz: 3cf95c22b05c8eaebdf646963d930977ddb010c3ca9174f081cee674e315f5ac672059b31f833ec337c78e3ec43770f72f6bba145ad00e2bd8c1ed8d0acb2174

data/.github/workflows/test.yml

@@ -22,6 +22,6 @@ jobs:
     - name: Install dependencies
       run: |
         gem install bundler
-        bundle install
+        POLYPHONY_USE_LIBEV=1 bundle install
     - name: Run tests
       run:  bundle exec rake test

data/.gitignore

@@ -54,3 +54,4 @@ build-iPhoneSimulator/
 
 # Used by RuboCop. Remote config files pulled in from inherit_from directive.
 # .rubocop-https?--*
+log

data/CHANGELOG.md

@@ -1,72 +1,96 @@
+## 0.40 2021-06-24
+
+- Implement serving static files using splice_chunks (nice performance boost for
+  files bigger than 1M)
+- Call shutdown before closing socket
+- Fix examples (thanks @timhatch!)
+
+## 0.39 2021-06-20
+
+- More work on DF server
+- Fix HTTP2StreamHandler#send_headers
+- Various fixes to HTTP/2 adapter
+- Fix host detection for HTTP/2 connections
+- Fix HTTP/1 adapter #respond with nil body
+- Fix HTTP1Adapter#send_headers
+
+## 0.38 2021-03-09
+
+- Don't use chunked transfer encoding for non-streaming responses
+
+## 0.37.2 2021-03-08
+
+- Fix header formatting when header value is an array
+
 ## 0.37 2021-02-15
 
-* Update upgrade mechanism to work with updated Qeweney API
+- Update upgrade mechanism to work with updated Qeweney API
 
 ## 0.36 2021-02-12
 
-* Use `Qeweney::Status` constants
+- Use `Qeweney::Status` constants
 
 ## 0.35 2021-02-10
 
-* Extract Request class into separate [qeweney](https://github.com/digital-fabric/qeweney) gem
+- Extract Request class into separate [qeweney](https://github.com/digital-fabric/qeweney) gem
 
 ## 0.34 2021-02-07
 
-* Implement digital fabric service and agents
-* Add multipart and urlencoded form data parsing
-* Improve request body reading behaviour
-* Add more `Request` information methods
-* Add access to connection for HTTP2 requests
-* Allow calling `Request#send_chunk` with empty chunk
-* Add support for handling protocol upgrades from within request handler
+- Implement digital fabric service and agents
+- Add multipart and urlencoded form data parsing
+- Improve request body reading behaviour
+- Add more `Request` information methods
+- Add access to connection for HTTP2 requests
+- Allow calling `Request#send_chunk` with empty chunk
+- Add support for handling protocol upgrades from within request handler
 
 ## 0.33 2020-11-20
 
-* Update code for Polyphony 0.47.5
-* Add support for Rack::File body to Tipi::RackAdapter
+- Update code for Polyphony 0.47.5
+- Add support for Rack::File body to Tipi::RackAdapter
 
 ## 0.32 2020-08-14
 
-* Respond with array of strings instead of concatenating for HTTP 1
-* Use read_loop instead of readpartial
-* Fix http upgrade test
+- Respond with array of strings instead of concatenating for HTTP 1
+- Use read_loop instead of readpartial
+- Fix http upgrade test
 
 ## 0.31 2020-07-28
 
-* Fix websocket server code
-* Implement configuration layer (WIP)
-* Improve performance of rack adapter
+- Fix websocket server code
+- Implement configuration layer (WIP)
+- Improve performance of rack adapter
 
 ## 0.30 2020-07-15
 
-* Rename project to Tipi
-* Rearrange source code
-* Remove HTTP client code (to be developed eventually into a separate gem)
-* Fix header rendering in rack adapter (#2)
+- Rename project to Tipi
+- Rearrange source code
+- Remove HTTP client code (to be developed eventually into a separate gem)
+- Fix header rendering in rack adapter (#2)
 
 ## 0.29 2020-07-06
 
-* Use IO#read_loop
+- Use IO#read_loop
 
 ## 0.28 2020-07-03
 
-* Update with API changes from Polyphony >= 0.41
+- Update with API changes from Polyphony >= 0.41
 
 ## 0.27 2020-04-14
 
-* Remove modulation dependency
+- Remove modulation dependency
 
 ## 0.26 2020-03-03
 
-* Fix `Server#listen`
+- Fix `Server#listen`
 
 ## 0.25 2020-02-19
 
-* Ensure server socket is closed upon stopping loop
-* Fix `Request#format_header_lines`
+- Ensure server socket is closed upon stopping loop
+- Fix `Request#format_header_lines`
 
 ## 0.24 2020-01-08
 
-* Move HTTP to separate polyphony-http gem
+- Move HTTP to separate polyphony-http gem
 
 For earlier changes look at the Polyphony changelog.

data/Gemfile.lock

@@ -1,22 +1,38 @@
 PATH
   remote: .
   specs:
-    tipi (0.37)
+    tipi (0.40)
+      acme-client (~> 2.0.8)
       http-2 (~> 0.10.0)
       http_parser.rb (~> 0.6.0)
       msgpack (~> 1.4.2)
-      polyphony (~> 0.51.0)
-      qeweney (~> 0.5)
+      polyphony (~> 0.57.0)
+      qeweney (~> 0.10.0)
       rack (>= 2.0.8, < 2.3.0)
       websocket (~> 1.2.8)
 
 GEM
   remote: https://rubygems.org/
   specs:
+    acme-client (2.0.8)
+      faraday (>= 0.17, < 2.0.0)
     ansi (1.5.0)
     builder (3.2.4)
     docile (1.3.2)
     escape_utils (1.2.1)
+    faraday (1.4.3)
+      faraday-em_http (~> 1.0)
+      faraday-em_synchrony (~> 1.0)
+      faraday-excon (~> 1.1)
+      faraday-net_http (~> 1.0)
+      faraday-net_http_persistent (~> 1.1)
+      multipart-post (>= 1.2, < 3)
+      ruby2_keywords (>= 0.0.4)
+    faraday-em_http (1.0.0)
+    faraday-em_synchrony (1.0.0)
+    faraday-excon (1.1.0)
+    faraday-net_http (1.0.1)
+    faraday-net_http_persistent (1.1.0)
     http-2 (0.10.2)
     http_parser.rb (0.6.0)
     json (2.3.1)
@@ -28,12 +44,14 @@ GEM
       minitest (>= 5.0)
       ruby-progressbar
     msgpack (1.4.2)
-    polyphony (0.51.0)
-    qeweney (0.5)
+    multipart-post (2.1.1)
+    polyphony (0.57.0)
+    qeweney (0.10)
       escape_utils (~> 1.2.1)
     rack (2.2.3)
     rake (12.3.3)
     ruby-progressbar (1.10.1)
+    ruby2_keywords (0.0.4)
     simplecov (0.17.1)
       docile (~> 1.1)
       json (>= 1.8, < 3)

data/TODO.md

@@ -1,8 +1,82 @@
-# Digital Fabric
+## Add an API for reading a request body chunk into an IO (pipe)
+
+      ```ruby
+      # currently
+      chunk = req.next_chunk
+      # or
+      req.each_chunk { |c| do_something(c) }
+
+      # what we'd like to do
+      r, w = IO.pipe
+      len = req.splice_chunk(w)
+      sock << "Here comes a chunk of #{len} bytes\n"
+      sock.splice(r, len)
+
+      # or:
+      r, w = IO.pipe
+      req.splice_each_chunk(w) do |len|
+        sock << "Here comes a chunk of #{len} bytes\n"
+        sock.splice(r, len)
+      end
+      ```
+
+# HTTP/1.1 parser
+
+- httparser.rb is not actively updated
+- the httparser.rb C parser code comes originally from https://github.com/nodejs/llhttp
+- there's a Ruby gem https://github.com/metabahn/llhttp, but its API is too low-level
+  (lots of callbacks, headers need to be retained across callbacks)
+- the basic idea is to import the C-code, then build a parser object with the following
+  callbacks:
+
+  ```ruby
+  on_headers_complete(headers)
+  on_body_chunk(chunk)
+  on_message_complete
+  ```
+
+- The llhttp gem's C-code is here: https://github.com/metabahn/llhttp/tree/main/mri
+
+- Actually, if you do a C extension, instead of a callback-based API, we can
+  design a blocking API:
+
+  ```ruby
+  parser = Tipi::HTTP1::Parser.new
+  parser.each_request(socket) do |headers|
+    request = Request.new(normalize_headers(headers))
+    handle_request(request)
+  end
+  ```
+
+# What about HTTP/2?
+
+It would be a nice exercise in converting a callback-based API to a blocking
+one:
+
+```ruby
+parser = Tipi::HTTP2::Parser.new(socket)
+parser.each_stream(socket) do |stream|
+  spin { handle_stream(stream) }
+end
+```
+
+
+
+# DF
+
+- Add attack protection for IP-address HTTP host:
+
+  ```ruby
+  IPV4_REGEXP = /^\d+\.\d+\.\d+\.\d+$/.freeze
+
+  def is_attack_request?(req)
+    return true if req.host =~ IPV4_REGEXP && req.query[:q] != 'ping'
+  end
+  ```
+
+- Add attack route to Qeweney routing API
 
-Problems to fix:
 
-- Memory leak (in server? multi agent? multi client?)
 
 # Roadmap
 
@@ -14,7 +88,7 @@ Problems to fix:
   - https://gitlab.com/honeyryderchuck/http-2-next
   - Open an issue there, ask what's the difference between the two gems?
 
-## 0.35
+## 0.38
 
 - Add more poly CLI commands and options:
 
@@ -26,7 +100,7 @@ Problems to fix:
   - set port to bind to
   - set forking process count
 
-## 0.36 Working Sinatra application
+## 0.39 Working Sinatra application
 
 - app with database access (postgresql)
 - benchmarks!

data/df/sample_agent.rb

@@ -13,7 +13,7 @@ class SampleAgent < DigitalFabric::Agent
   HTML_SSE = IO.read(File.join(__dir__, 'sse_page.html'))
 
   def http_request(req)
-    path = req['headers'][':path']
+    path = req.headers[':path']
     case path
     when '/agent'
       send_df_message(Protocol.http_response(

data/df/server.rb

@@ -6,12 +6,17 @@ require 'tipi/digital_fabric'
 require 'tipi/digital_fabric/executive'
 require 'json'
 require 'fileutils'
+require 'localhost/authority'
+
 FileUtils.cd(__dir__)
 
 service = DigitalFabric::Service.new(token: 'foobar')
 executive = DigitalFabric::Executive.new(service, { host: 'executive.realiteq.net' })
 
-spin_loop(interval: 60) { GC.start }
+GC.disable
+Thread.current.backend.idle_gc_period = 60
+
+# spin_loop(interval: 60) { GC.start }
 
 class Polyphony::BaseException
   attr_reader :caller_backtrace
@@ -19,13 +24,13 @@ end
 
 puts "pid: #{Process.pid}"
 
-tcp_listener = spin do
+http_listener = spin do
   opts = {
     reuse_addr:  true,
     dont_linger: true,
   }
-  puts 'Listening on localhost:4411'
-  server = Polyphony::Net.tcp_listen('0.0.0.0', 4411, opts)
+  puts 'Listening for HTTP on localhost:10080'
+  server = Polyphony::Net.tcp_listen('0.0.0.0', 10080, opts)
   server.accept_loop do |client|
     spin do
       service.incr_connection_count
@@ -33,11 +38,59 @@ tcp_listener = spin do
     ensure
       service.decr_connection_count
     end
+  rescue Exception => e
+    puts "HTTP accept_loop error: #{e.inspect}"
+    puts e.backtrace.join("\n")
   end
 end
 
-UNIX_SOCKET_PATH = '/tmp/df.sock'
+CERTIFICATE_REGEXP = /(-----BEGIN CERTIFICATE-----\n[^-]+-----END CERTIFICATE-----\n)/.freeze
+
+https_listener = spin do
+  private_key = OpenSSL::PKey::RSA.new IO.read('../../reality/ssl/privkey.pem')
+  c = IO.read('../../reality/ssl/cacert.pem')
+  certificates = c.scan(CERTIFICATE_REGEXP).map { |p|  OpenSSL::X509::Certificate.new(p.first) }
+  ctx = OpenSSL::SSL::SSLContext.new
+  cert = certificates.shift
+  puts "Certificate expires: #{cert.not_after.inspect}"
+  ctx.add_certificate(cert, private_key, certificates)
+  ctx.ciphers = 'ECDH+aRSA'
+
+  # TODO: further limit ciphers
+  # ref: https://github.com/socketry/falcon/blob/3ec805b3ceda0a764a2c5eb68cde33897b6a35ff/lib/falcon/environments/tls.rb
+  # ref: https://github.com/socketry/falcon/blob/3ec805b3ceda0a764a2c5eb68cde33897b6a35ff/lib/falcon/tls.rb
+
+  opts = {
+    reuse_addr:     true,
+    dont_linger:    true,
+    secure_context: ctx,
+    alpn_protocols: Tipi::ALPN_PROTOCOLS
+  }
+
+  puts 'Listening for HTTPS on localhost:10443'
+  server = Polyphony::Net.tcp_listen('0.0.0.0', 10443, opts)
+  loop do
+    client = server.accept
+    spin do
+      service.incr_connection_count
+      Tipi.client_loop(client, opts) { |req| service.http_request(req) }
+    rescue Exception => e
+      puts "Exception: #{e.inspect}"
+      puts e.backtrace.join("\n")
+    ensure
+      service.decr_connection_count
+    end
+  rescue Polyphony::BaseException
+    raise
+  rescue OpenSSL::SSL::SSLError, SystemCallError => e
+    puts "HTTPS accept error: #{e.inspect}"
+  rescue Exception => e
+    puts "HTTPS accept error: #{e.inspect}"
+    puts e.backtrace.join("\n")
+  end
+end
 
+UNIX_SOCKET_PATH = '/tmp/df.sock'
 unix_listener = spin do
   puts "Listening on #{UNIX_SOCKET_PATH}"
   FileUtils.rm(UNIX_SOCKET_PATH) if File.exists?(UNIX_SOCKET_PATH)
@@ -46,9 +99,13 @@ unix_listener = spin do
 end
 
 begin
-  Fiber.await(tcp_listener, unix_listener)
+  Fiber.await(http_listener, https_listener, unix_listener)
 rescue Interrupt
   puts "Got SIGINT, shutting down gracefully"
   service.graceful_shutdown
   puts "post graceful shutdown"
+rescue Exception => e
+  puts '*' * 40
+  p e
+  puts e.backtrace.join("\n")
 end

data/examples/automatic_certificate.rb

@@ -0,0 +1,193 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'tipi'
+require 'openssl'
+require 'acme-client'
+
+# ::Exception.__disable_sanitized_backtrace__ = true
+
+class CertificateManager
+  def initialize(store:, challenge_handler:)
+    @store = store
+    @challenge_handler = challenge_handler
+    @workers = {}
+    @contexts = {}
+  end
+
+  def [](name)
+    worker = worker_for_name(name)
+    p worker: worker
+  
+    worker << Fiber.current
+    # cancel_after(30) { receive }
+    receive.tap { |ctx| p got_ctx: ctx }
+  rescue Exception => e
+    p e
+    puts e.backtrace.join("\n")
+    nil
+  end
+  
+  def worker_for_name(name)
+    @workers[name] ||= spin { worker_loop(name) }
+  end
+  
+  def worker_loop(name)
+    while (client = receive)
+      puts "get request for #{name} from #{client.inspect}"
+      ctx = get_context(name)
+      client << ctx rescue nil
+    end
+  end
+
+  def get_context(name)
+    @contexts[name] ||= setup_context(name)
+  end
+
+  CERTIFICATE_REGEXP = /(-----BEGIN CERTIFICATE-----\n[^-]+-----END CERTIFICATE-----\n)/.freeze
+
+  def setup_context(name)
+    certificate = get_certificate(name)
+    ctx = OpenSSL::SSL::SSLContext.new
+    chain = certificate.scan(CERTIFICATE_REGEXP).map { |p|  OpenSSL::X509::Certificate.new(p.first) }
+    cert = chain.shift
+    puts "Certificate expires: #{cert.not_after.inspect}"
+    ctx.add_certificate(cert, private_key, chain)
+    Polyphony::Net.setup_alpn(ctx, Tipi::ALPN_PROTOCOLS)
+    ctx
+  end
+  
+  def get_certificate(name)
+    @store[name] ||= provision_certificate(name)
+  end
+
+  def private_key
+    @private_key ||= OpenSSL::PKey::RSA.new(4096)
+  end
+
+  ACME_DIRECTORY = 'https://acme-staging-v02.api.letsencrypt.org/directory'
+
+  def acme_client
+    @acme_client ||= setup_acme_client
+  end
+
+  def setup_acme_client
+    client = Acme::Client.new(
+      private_key: private_key,
+      directory: ACME_DIRECTORY
+    )
+    p client: client
+    account = client.new_account(
+      contact: 'mailto:info@noteflakes.com',
+      terms_of_service_agreed: true
+    )
+    p account: account.kid
+    client
+  end
+
+  def provision_certificate(name)
+    order = acme_client.new_order(identifiers: [name])
+    p order: true
+    authorization = order.authorizations.first
+    p authorization: authorization
+    challenge = authorization.http
+    p challenge: challenge
+  
+    @challenge_handler.add(challenge)
+    challenge.request_validation
+    p challenge_status: challenge.status
+    while challenge.status == 'pending'
+      sleep(1)
+      challenge.reload
+      p challenge_status: challenge.status
+    end
+  
+    csr = Acme::Client::CertificateRequest.new(private_key: @private_key, subject: { common_name: name })
+    p csr: csr
+    order.finalize(csr: csr)
+    p order_status: order.status
+    while order.status == 'processing'
+      sleep(1)
+      order.reload
+      p order_status: order.status
+    end
+    order.certificate.tap { |c| p certificate: c } # => PEM-formatted certificate
+  end
+end
+
+class AcmeHTTPChallengeHandler
+  def initialize
+    @challenges = {}
+  end
+
+  def add(challenge)
+    path = "/.well-known/acme-challenge/#{challenge.token}"
+    @challenges[path] = challenge
+  end
+
+  def call(req)
+    # handle incoming request
+    challenge = @challenges[req.path]
+    return req.respond(nil, ':status' => 400) unless challenge
+
+    req.respond(challenge.file_content, 'content-type' => challenge.content_type)
+    @challenges.delete(req.path)
+  end
+end
+
+challenge_handler = AcmeHTTPChallengeHandler.new
+certificate_manager = CertificateManager.new(
+  store: {},
+  challenge_handler: challenge_handler
+)
+
+http_handler = Tipi.route do |r|
+  r.on('/.well-known/acme-challenge') { challenge_handler.call(r) }
+  r.default { r.redirect "https://#{r.host}#{r.path}" }
+end
+
+https_handler = ->(r) { r.respond('Hello, world!') }
+
+http_listener = spin do
+  opts = {
+    reuse_addr:  true,
+    dont_linger: true,
+  }
+  puts 'Listening for HTTP on localhost:10080'
+  Tipi.serve('0.0.0.0', 10080, opts, &http_handler)
+end
+
+https_listener = spin do
+  ctx = OpenSSL::SSL::SSLContext.new
+  ctx.servername_cb = proc { |_, name| p name: name; certificate_manager[name] }
+  opts = {
+    reuse_addr:     true,
+    dont_linger:    true,
+    secure_context: ctx,
+    alpn_protocols: Tipi::ALPN_PROTOCOLS
+  }
+
+  puts 'Listening for HTTPS on localhost:10443'
+  server = Polyphony::Net.tcp_listen('0.0.0.0', 10443, opts)
+  server.accept_loop do |client|
+    spin do
+      service.incr_connection_count
+      Tipi.client_loop(client, opts) { |req| service.http_request(req) }
+    ensure
+      service.decr_connection_count
+    end
+  rescue Exception => e
+    puts "HTTPS accept_loop error: #{e.inspect}"
+    puts e.backtrace.join("\n")
+  end
+end
+
+begin
+  Fiber.await(http_listener, https_listener)
+rescue Interrupt
+  puts "Got SIGINT, terminating"
+rescue Exception => e
+  puts '*' * 40
+  p e
+  puts e.backtrace.join("\n")
+end