tipi 0.36 → 0.39

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 82a6e09e9b1777016ef5da412c7e60d7224c8ea4d2839b486f02ccda2774d522
-  data.tar.gz: 0d2ae87d9c2525432417699553670503528e52a6ca3c0d400e8a7a0bfeae0142
+  metadata.gz: 02c5ce4798e1961ca59798b75bafe158570e777661f6fc8667169553df3cd3f4
+  data.tar.gz: 405bfa3526efaad394d34840764dc131230a366574cfc52b63344a6d5cdfe6dd
 SHA512:
-  metadata.gz: db5d80d15a5fc164b09461fe91bb0a35549910f6e7783583f0df838c3d5d5379b797cb67bd7d19236b9a5388a2ba1957fa081092ca103643cd028d475668cfcd
-  data.tar.gz: f9d965dcc195568900d3a772892925bea90fc45b8e8a5e8ad4c0370323a388b26c66e6a85fc629c73d9b028caebc4f6bf27c430991ba9e1e2dd4335fce98bc6a
+  metadata.gz: 754bd35136e4e004e6bd782eb2060d933b2439c437a9c2a966529e0465c42096b97fa08b1dae04b7a3c8aa0c77c367fd5395d215cca73fd7cf1aa80c297b3178
+  data.tar.gz: 24425d798c076ad43b4fa45b641a90a5e801dfe35acd6217e8a8fc97ea546f18457fc9a8653e47f7bd5f696da885ef86db8db36385dcaa79ccbcf9d42d64081c

data/.github/workflows/test.yml

@@ -22,6 +22,6 @@ jobs:
     - name: Install dependencies
       run: |
         gem install bundler
-        bundle install
+        POLYPHONY_USE_LIBEV=1 bundle install
     - name: Run tests
       run:  bundle exec rake test

data/.gitignore

@@ -54,3 +54,4 @@ build-iPhoneSimulator/
 
 # Used by RuboCop. Remote config files pulled in from inherit_from directive.
 # .rubocop-https?--*
+log

data/CHANGELOG.md

@@ -1,68 +1,89 @@
+## 0.39 2021-06-20
+
+- More work on DF server
+- Fix HTTP2StreamHandler#send_headers
+- Various fixes to HTTP/2 adapter
+- Fix host detection for HTTP/2 connections
+- Fix HTTP/1 adapter #respond with nil body
+- Fix HTTP1Adapter#send_headers
+
+## 0.38 2021-03-09
+
+- Don't use chunked transfer encoding for non-streaming responses
+
+## 0.37.2 2021-03-08
+
+- Fix header formatting when header value is an array
+
+## 0.37 2021-02-15
+
+- Update upgrade mechanism to work with updated Qeweney API
+
 ## 0.36 2021-02-12
 
-* Use `Qeweney::Status` constants
+- Use `Qeweney::Status` constants
 
 ## 0.35 2021-02-10
 
-* Extract Request class into separate [qeweney](https://github.com/digital-fabric/qeweney) gem
+- Extract Request class into separate [qeweney](https://github.com/digital-fabric/qeweney) gem
 
 ## 0.34 2021-02-07
 
-* Implement digital fabric service and agents
-* Add multipart and urlencoded form data parsing
-* Improve request body reading behaviour
-* Add more `Request` information methods
-* Add access to connection for HTTP2 requests
-* Allow calling `Request#send_chunk` with empty chunk
-* Add support for handling protocol upgrades from within request handler
+- Implement digital fabric service and agents
+- Add multipart and urlencoded form data parsing
+- Improve request body reading behaviour
+- Add more `Request` information methods
+- Add access to connection for HTTP2 requests
+- Allow calling `Request#send_chunk` with empty chunk
+- Add support for handling protocol upgrades from within request handler
 
 ## 0.33 2020-11-20
 
-* Update code for Polyphony 0.47.5
-* Add support for Rack::File body to Tipi::RackAdapter
+- Update code for Polyphony 0.47.5
+- Add support for Rack::File body to Tipi::RackAdapter
 
 ## 0.32 2020-08-14
 
-* Respond with array of strings instead of concatenating for HTTP 1
-* Use read_loop instead of readpartial
-* Fix http upgrade test
+- Respond with array of strings instead of concatenating for HTTP 1
+- Use read_loop instead of readpartial
+- Fix http upgrade test
 
 ## 0.31 2020-07-28
 
-* Fix websocket server code
-* Implement configuration layer (WIP)
-* Improve performance of rack adapter
+- Fix websocket server code
+- Implement configuration layer (WIP)
+- Improve performance of rack adapter
 
 ## 0.30 2020-07-15
 
-* Rename project to Tipi
-* Rearrange source code
-* Remove HTTP client code (to be developed eventually into a separate gem)
-* Fix header rendering in rack adapter (#2)
+- Rename project to Tipi
+- Rearrange source code
+- Remove HTTP client code (to be developed eventually into a separate gem)
+- Fix header rendering in rack adapter (#2)
 
 ## 0.29 2020-07-06
 
-* Use IO#read_loop
+- Use IO#read_loop
 
 ## 0.28 2020-07-03
 
-* Update with API changes from Polyphony >= 0.41
+- Update with API changes from Polyphony >= 0.41
 
 ## 0.27 2020-04-14
 
-* Remove modulation dependency
+- Remove modulation dependency
 
 ## 0.26 2020-03-03
 
-* Fix `Server#listen`
+- Fix `Server#listen`
 
 ## 0.25 2020-02-19
 
-* Ensure server socket is closed upon stopping loop
-* Fix `Request#format_header_lines`
+- Ensure server socket is closed upon stopping loop
+- Fix `Request#format_header_lines`
 
 ## 0.24 2020-01-08
 
-* Move HTTP to separate polyphony-http gem
+- Move HTTP to separate polyphony-http gem
 
 For earlier changes look at the Polyphony changelog.

data/Gemfile.lock

@@ -1,12 +1,12 @@
 PATH
   remote: .
   specs:
-    tipi (0.36)
+    tipi (0.39)
       http-2 (~> 0.10.0)
       http_parser.rb (~> 0.6.0)
       msgpack (~> 1.4.2)
-      polyphony (~> 0.51.0)
-      qeweney (~> 0.3)
+      polyphony (~> 0.55.0)
+      qeweney (~> 0.9.1)
       rack (>= 2.0.8, < 2.3.0)
       websocket (~> 1.2.8)
 
@@ -28,8 +28,8 @@ GEM
       minitest (>= 5.0)
       ruby-progressbar
     msgpack (1.4.2)
-    polyphony (0.51.0)
-    qeweney (0.3)
+    polyphony (0.55.0)
+    qeweney (0.9.1)
       escape_utils (~> 1.2.1)
     rack (2.2.3)
     rake (12.3.3)

data/TODO.md

@@ -1,8 +1,82 @@
-# Digital Fabric
+## Add an API for reading a request body chunk into an IO (pipe)
+
+      ```ruby
+      # currently
+      chunk = req.next_chunk
+      # or
+      req.each_chunk { |c| do_something(c) }
+
+      # what we'd like to do
+      r, w = IO.pipe
+      len = req.splice_chunk(w)
+      sock << "Here comes a chunk of #{len} bytes\n"
+      sock.splice(r, len)
+
+      # or:
+      r, w = IO.pipe
+      req.splice_each_chunk(w) do |len|
+        sock << "Here comes a chunk of #{len} bytes\n"
+        sock.splice(r, len)
+      end
+      ```
+
+# HTTP/1.1 parser
+
+- httparser.rb is not actively updated
+- the httparser.rb C parser code comes originally from https://github.com/nodejs/llhttp
+- there's a Ruby gem https://github.com/metabahn/llhttp, but its API is too low-level
+  (lots of callbacks, headers need to be retained across callbacks)
+- the basic idea is to import the C-code, then build a parser object with the following
+  callbacks:
+
+  ```ruby
+  on_headers_complete(headers)
+  on_body_chunk(chunk)
+  on_message_complete
+  ```
+
+- The llhttp gem's C-code is here: https://github.com/metabahn/llhttp/tree/main/mri
+
+- Actually, if you do a C extension, instead of a callback-based API, we can
+  design a blocking API:
+
+  ```ruby
+  parser = Tipi::HTTP1::Parser.new
+  parser.each_request(socket) do |headers|
+    request = Request.new(normalize_headers(headers))
+    handle_request(request)
+  end
+  ```
+
+# What about HTTP/2?
+
+It would be a nice exercise in converting a callback-based API to a blocking
+one:
+
+```ruby
+parser = Tipi::HTTP2::Parser.new(socket)
+parser.each_stream(socket) do |stream|
+  spin { handle_stream(stream) }
+end
+```
+
+
+
+# DF
+
+- Add attack protection for IP-address HTTP host:
+
+  ```ruby
+  IPV4_REGEXP = /^\d+\.\d+\.\d+\.\d+$/.freeze
+
+  def is_attack_request?(req)
+    return true if req.host =~ IPV4_REGEXP && req.query[:q] != 'ping'
+  end
+  ```
+
+- Add attack route to Qeweney routing API
 
-Problems to fix:
 
-- Memory leak (in server? multi agent? multi client?)
 
 # Roadmap
 
@@ -14,7 +88,7 @@ Problems to fix:
   - https://gitlab.com/honeyryderchuck/http-2-next
   - Open an issue there, ask what's the difference between the two gems?
 
-## 0.35
+## 0.38
 
 - Add more poly CLI commands and options:
 
@@ -26,7 +100,7 @@ Problems to fix:
   - set port to bind to
   - set forking process count
 
-## 0.36 Working Sinatra application
+## 0.39 Working Sinatra application
 
 - app with database access (postgresql)
 - benchmarks!

data/df/sample_agent.rb

@@ -13,7 +13,7 @@ class SampleAgent < DigitalFabric::Agent
   HTML_SSE = IO.read(File.join(__dir__, 'sse_page.html'))
 
   def http_request(req)
-    path = req['headers'][':path']
+    path = req.headers[':path']
     case path
     when '/agent'
       send_df_message(Protocol.http_response(

data/df/server.rb

@@ -6,6 +6,8 @@ require 'tipi/digital_fabric'
 require 'tipi/digital_fabric/executive'
 require 'json'
 require 'fileutils'
+require 'localhost/authority'
+
 FileUtils.cd(__dir__)
 
 service = DigitalFabric::Service.new(token: 'foobar')
@@ -19,13 +21,46 @@ end
 
 puts "pid: #{Process.pid}"
 
-tcp_listener = spin do
+http_listener = spin do
   opts = {
     reuse_addr:  true,
     dont_linger: true,
   }
-  puts 'Listening on localhost:4411'
-  server = Polyphony::Net.tcp_listen('0.0.0.0', 4411, opts)
+  puts 'Listening for HTTP on localhost:10080'
+  server = Polyphony::Net.tcp_listen('0.0.0.0', 10080, opts)
+  server.accept_loop do |client|
+    spin do
+      service.incr_connection_count
+      Tipi.client_loop(client, opts) { |req| service.http_request(req) }
+    ensure
+      service.decr_connection_count
+    end
+  rescue Exception => e
+    puts "HTTP accept_loop error: #{e.inspect}"
+    puts e.backtrace.join("\n")
+  end
+end
+
+CERTIFICATE_REGEXP = /(-----BEGIN CERTIFICATE-----\n[^-]+-----END CERTIFICATE-----\n)/.freeze
+
+https_listener = spin do
+  private_key = OpenSSL::PKey::RSA.new IO.read('../../reality/ssl/privkey.pem')
+  c = IO.read('../../reality/ssl/cacert.pem')
+  certificates = c.scan(CERTIFICATE_REGEXP).map { |p|  OpenSSL::X509::Certificate.new(p.first) }
+  ctx = OpenSSL::SSL::SSLContext.new
+  cert = certificates.shift
+  puts "Certificate expires: #{cert.not_after.inspect}"
+  ctx.add_certificate(cert, private_key, certificates)
+  # ctx = Localhost::Authority.fetch.server_context
+  opts = {
+    reuse_addr:     true,
+    dont_linger:    true,
+    secure_context: ctx,
+    alpn_protocols: Tipi::ALPN_PROTOCOLS
+  }
+
+  puts 'Listening for HTTPS on localhost:10443'
+  server = Polyphony::Net.tcp_listen('0.0.0.0', 10443, opts)
   server.accept_loop do |client|
     spin do
       service.incr_connection_count
@@ -33,6 +68,9 @@ tcp_listener = spin do
     ensure
       service.decr_connection_count
     end
+  rescue Exception => e
+    puts "HTTPS accept_loop error: #{e.inspect}"
+    puts e.backtrace.join("\n")
   end
 end
 
@@ -46,9 +84,13 @@ unix_listener = spin do
 end
 
 begin
-  Fiber.await(tcp_listener, unix_listener)
+  Fiber.await(http_listener, https_listener, unix_listener)
 rescue Interrupt
   puts "Got SIGINT, shutting down gracefully"
   service.graceful_shutdown
   puts "post graceful shutdown"
+rescue Exception => e
+  puts '*' * 40
+  p e
+  puts e.backtrace.join("\n")
 end

data/examples/http_request_ws_server.rb

@@ -27,7 +27,8 @@ puts 'Listening on port 4411...'
 
 Tipi.serve('0.0.0.0', 4411, opts) do |req|
   if req.upgrade_protocol == 'websocket'
-    ws_handler(Tipi::Websocket.new(req.adapter.conn, req.headers))
+    conn = req.upgrade_to_websocket
+    ws_handler(conn)
   else
     req.respond(HTML, 'Content-Type' => 'text/html')
   end

data/examples/http_server.rb

@@ -13,9 +13,17 @@ puts 'Listening on port 4411...'
 
 spin do
   Tipi.serve('0.0.0.0', 4411, opts) do |req|
-    req.respond("Hello world!\n")
-  rescue Exception => e
-    p e
+    if req.path == '/stream'
+      req.send_headers('Foo' => 'Bar')
+      sleep 1
+      req.send_chunk("foo\n")
+      sleep 1
+      req.send_chunk("bar\n")
+      req.finish
+    else
+      req.respond("Hello world!\n")
+    end
+    p req.transfer_counts
   end
   p 'done...'
 end.await

data/examples/http_server_forked.rb

@@ -25,4 +25,6 @@ end
 
 puts 'Listening on port 1234'
 
+trap('SIGINT') { exit! }
+
 child_pids.each { |pid| Thread.current.backend.waitpid(pid) }

data/examples/http_server_throttled.rb

@@ -3,9 +3,9 @@
 require 'bundler/setup'
 require 'tipi'
 
-$throttler = throttle(1000)
+$throttler = Polyphony::Throttler.new(1000)
 opts = { reuse_addr: true, dont_linger: true }
-spin do
+server = spin do
   Tipi.serve('0.0.0.0', 1234, opts) do |req|
     $throttler.call { req.respond("Hello world!\n") }
   end
@@ -13,3 +13,4 @@ end
 
 puts "pid: #{Process.pid}"
 puts 'Listening on port 1234...'
+server.await

data/examples/https_server.rb

@@ -16,7 +16,16 @@ opts = {
 puts "pid: #{Process.pid}"
 puts 'Listening on port 1234...'
 Tipi.serve('0.0.0.0', 1234, opts) do |req|
-  req.respond("Hello world!\n")
+  p path: req.path
+  if req.path == '/stream'
+    req.send_headers('Foo' => 'Bar')
+    sleep 0.5
+    req.send_chunk("foo\n")
+    sleep 0.5
+    req.send_chunk("bar\n", done: true)
+  else
+    req.respond("Hello world!\n")
+  end
   # req.send_headers
   # req.send_chunk("Method: #{req.method}\n")
   # req.send_chunk("Path: #{req.path}\n")