tipi 0.39 → 0.43

data/df/server_utils.rb

@@ -0,0 +1,175 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'tipi'
+require 'tipi/digital_fabric'
+require 'tipi/digital_fabric/executive'
+require 'json'
+require 'fileutils'
+require 'time'
+require 'polyphony/extensions/debug'
+
+FileUtils.cd(__dir__)
+
+@service = DigitalFabric::Service.new(token: 'foobar')
+@executive = DigitalFabric::Executive.new(@service, { host: '@executive.realiteq.net' })
+
+@pid = Process.pid
+
+def log(msg, **ctx)
+  text = format(
+    "%s (%d) %s\n",
+    Time.now.strftime('%Y-%m-%d %H:%M:%S.%3N'),
+    @pid,
+    msg
+  )
+  STDOUT.orig_write text
+  return if ctx.empty?
+
+  ctx.each { |k, v| STDOUT.orig_write format("  %s: %s\n", k, v.inspect) }
+end
+
+def listen_http
+  spin(:http_listener) do
+    opts = {
+      reuse_addr:  true,
+      dont_linger: true,
+    }
+    log('Listening for HTTP on localhost:10080')
+    server = Polyphony::Net.tcp_listen('0.0.0.0', 10080, opts)
+    id = 0
+    loop do
+      client = server.accept
+      # log("Accept HTTP connection", client: client)
+      spin("http#{id += 1}") do
+        @service.incr_connection_count
+        Tipi.client_loop(client, opts) { |req| @service.http_request(req) }
+      ensure
+        # log("Done with HTTP connection", client: client)
+        @service.decr_connection_count
+      end
+    rescue Polyphony::BaseException
+      raise
+    rescue Exception => e
+      log 'HTTP accept (unknown) error', error: e, backtrace: e.backtrace
+    end
+  end
+end
+
+CERTIFICATE_REGEXP = /(-----BEGIN CERTIFICATE-----\n[^-]+-----END CERTIFICATE-----\n)/.freeze
+
+def listen_https
+  spin(:https_listener) do
+    private_key = OpenSSL::PKey::RSA.new IO.read('../../reality/ssl/privkey.pem')
+    c = IO.read('../../reality/ssl/cacert.pem')
+    certificates = c.scan(CERTIFICATE_REGEXP).map { |p|  OpenSSL::X509::Certificate.new(p.first) }
+    ctx = OpenSSL::SSL::SSLContext.new
+    ctx.security_level = 0
+    cert = certificates.shift
+    log "SSL Certificate expires: #{cert.not_after.inspect}"
+    ctx.add_certificate(cert, private_key, certificates)
+    ctx.ciphers = 'ECDH+aRSA'
+    ctx.max_version = OpenSSL::SSL::TLS1_3_VERSION
+    ctx.min_version = OpenSSL::SSL::SSL3_VERSION
+    ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE
+
+    # TODO: further limit ciphers
+    # ref: https://github.com/socketry/falcon/blob/3ec805b3ceda0a764a2c5eb68cde33897b6a35ff/lib/falcon/environments/tls.rb
+    # ref: https://github.com/socketry/falcon/blob/3ec805b3ceda0a764a2c5eb68cde33897b6a35ff/lib/falcon/tls.rb
+
+    opts = {
+      reuse_addr:     true,
+      dont_linger:    true,
+      secure_context: ctx,
+      alpn_protocols: Tipi::ALPN_PROTOCOLS
+    }
+
+    log('Listening for HTTPS on localhost:10443')
+    server = Polyphony::Net.tcp_listen('0.0.0.0', 10443, opts)
+    id = 0
+    loop do
+      client = server.accept rescue nil
+      next unless client
+
+      # log('Accept HTTPS client connection', client: client)
+      spin("https#{id += 1}") do
+        @service.incr_connection_count
+        Tipi.client_loop(client, opts) { |req| @service.http_request(req) }
+      rescue => e
+        log('Error while handling HTTPS client', client: client, error: e, backtrace: e.backtrace)
+      ensure
+        # log("Done with HTTP connection", client: client)
+        @service.decr_connection_count
+      end
+    # rescue OpenSSL::SSL::SSLError, SystemCallError, TypeError => e
+    #   log('HTTPS accept error', error: e)
+    rescue Polyphony::BaseException
+      raise
+    rescue Exception => e
+      log 'HTTPS listener error: ', error: e, backtrace: e.backtrace
+    end
+  end
+end
+
+UNIX_SOCKET_PATH = '/tmp/df.sock'
+def listen_unix
+  spin(:unix_listener) do
+    log("Listening on #{UNIX_SOCKET_PATH}")
+    FileUtils.rm(UNIX_SOCKET_PATH) if File.exists?(UNIX_SOCKET_PATH)
+    socket = UNIXServer.new(UNIX_SOCKET_PATH)
+
+    id = 0
+    loop do
+      client = socket.accept
+      # log('Accept Unix connection', client: client)
+      spin("unix#{id += 1}") do
+        Tipi.client_loop(client, {}) { |req| @service.http_request(req, true) }
+      end
+    rescue Polyphony::BaseException
+      raise
+    rescue Exception => e
+      log 'Unix accept error', error: e, backtrace: e.backtrace
+    end
+  end
+end
+
+def listen_df
+  spin(:df_listener) do
+    opts = {
+      reuse_addr:  true,
+      reuse_port:  true,
+      dont_linger: true,
+    }
+    log('Listening for DF connections on localhost:4321')
+    server = Polyphony::Net.tcp_listen('0.0.0.0', 4321, opts)
+
+    id = 0
+    loop do
+      client = server.accept
+      # log('Accept DF connection', client: client)
+      spin("df#{id += 1}") do
+        Tipi.client_loop(client, {}) { |req| @service.http_request(req, true) }
+      end
+    rescue Polyphony::BaseException
+      raise
+    rescue Exception => e
+      log 'DF accept (unknown) error', error: e, backtrace: e.backtrace
+    end
+  end
+end
+
+if ENV['TRACE'] == '1'
+  Thread.backend.trace_proc = proc do |event, fiber, value, pri|
+    fiber_id = fiber.tag || fiber.inspect
+    case event
+    when :fiber_schedule
+      log format("=> %s %s %s %s", event, fiber_id, value.inspect, pri ? '(priority)' : '')
+    when :fiber_run
+      log format("=> %s %s %s", event, fiber_id, value.inspect)
+    when :fiber_create, :fiber_terminate
+      log format("=> %s %s", event, fiber_id)
+    else
+      log format("=> %s", event)
+    end
+  end
+end

data/examples/full_service.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'tipi'
+
+::Exception.__disable_sanitized_backtrace__ = true
+
+certificate_db_path = File.expand_path('certificate_store.db', __dir__)
+certificate_store = Tipi::ACME::SQLiteCertificateStore.new(certificate_db_path)
+
+Tipi.full_service(
+  certificate_store: certificate_store
+) { |req| req.respond('Hello, world!') }

data/examples/http1_parser.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require 'polyphony'
+require_relative '../lib/tipi_ext'
+
+i, o = IO.pipe
+
+module ::Kernel
+  def trace(*args)
+    STDOUT.orig_write(format_trace(args))
+  end
+
+  def format_trace(args)
+    if args.first.is_a?(String)
+      if args.size > 1
+        format("%s: %p\n", args.shift, args)
+      else
+        format("%s\n", args.first)
+      end
+    else
+      format("%p\n", args.size == 1 ? args.first : args)
+    end
+  end
+end
+
+f = spin do
+  parser = Tipi::HTTP1Parser.new(i)
+  while true
+    trace '*' * 40
+    headers = parser.parse_headers
+    break unless headers
+    trace headers
+
+    body = parser.read_body
+    trace "body: #{body ? body.bytesize : 0} bytes"
+    trace body if body && body.bytesize < 80
+  end
+end
+
+o << "GET /a HTTP/1.1\r\n\r\n"
+
+# o << "GET /a HTTP/1.1\r\nContent-Length: 0\r\n\r\n"
+
+# o << "GET / HTTP/1.1\r\nHost: localhost:10080\r\nUser-Agent: curl/7.74.0\r\nAccept: */*\r\n\r\n"
+
+o << "post /?q=time&blah=blah HTTP/1\r\nTransfer-Encoding: chunked\r\n\r\na\r\nabcdefghij\r\n0\r\n\r\n"
+
+data = " " * 4000000
+o << "get /?q=time HTTP/1.1\r\nContent-Length: #{data.bytesize}\r\n\r\n#{data}"
+
+o << "get /?q=time HTTP/1.1\r\nCookie: foo\r\nCookie: bar\r\n\r\n"
+
+o.close
+
+f.await

data/examples/http_server.rb

@@ -9,10 +9,19 @@ opts = {
 }
 
 puts "pid: #{Process.pid}"
-puts 'Listening on port 4411...'
+puts 'Listening on port 10080...'
+
+# GC.disable
+# Thread.current.backend.idle_gc_period = 60
+
+spin_loop(interval: 10) { p Thread.backend.stats }
+
+spin_loop(interval: 10) do
+  GC.compact
+end
 
 spin do
-  Tipi.serve('0.0.0.0', 4411, opts) do |req|
+  Tipi.serve('0.0.0.0', 10080, opts) do |req|
     if req.path == '/stream'
       req.send_headers('Foo' => 'Bar')
       sleep 1
@@ -20,10 +29,13 @@ spin do
       sleep 1
       req.send_chunk("bar\n")
       req.finish
+    elsif req.path == '/upload'
+      body = req.read
+      req.respond("Body: #{body.inspect} (#{body.bytesize} bytes)")
     else
       req.respond("Hello world!\n")
     end
-    p req.transfer_counts
+#    p req.transfer_counts
   end
   p 'done...'
 end.await

data/examples/http_server_forked.rb

@@ -11,11 +11,13 @@ opts = {
   dont_linger: true
 }
 
+server = Tipi.listen('0.0.0.0', 1234, opts)
+
 child_pids = []
 8.times do
   pid = Polyphony.fork do
     puts "forked pid: #{Process.pid}"
-    Tipi.serve('0.0.0.0', 1234, opts) do |req|
+    server.each do |req|
       req.respond("Hello world! from pid: #{Process.pid}\n")
     end
   rescue Interrupt

data/examples/http_server_routes.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'tipi'
+
+opts = {
+  reuse_addr:  true,
+  dont_linger: true
+}
+
+puts "pid: #{Process.pid}"
+puts 'Listening on port 4411...'
+
+app = Tipi.route do |req|
+  req.on 'stream' do
+    req.send_headers('Foo' => 'Bar')
+    sleep 1
+    req.send_chunk("foo\n")
+    sleep 1
+    req.send_chunk("bar\n")
+    req.finish
+  end
+  req.default do
+    req.respond("Hello world!\n")
+  end
+end
+
+trap('INT') { exit! }
+Tipi.serve('0.0.0.0', 4411, opts, &app)

data/examples/http_server_static.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'tipi'
+require 'fileutils'
+
+opts = {
+  reuse_addr:  true,
+  dont_linger: true
+}
+
+puts "pid: #{Process.pid}"
+puts 'Listening on port 4411...'
+
+root_path = FileUtils.pwd
+
+trap('INT') { exit! }
+
+Tipi.serve('0.0.0.0', 4411, opts) do |req|
+  path = File.join(root_path, req.path)
+  if File.file?(path)
+    req.serve_file(path)
+  else
+    req.respond(nil, ':status' => Qeweney::Status::NOT_FOUND)
+  end
+end

data/examples/https_server.rb

@@ -23,6 +23,9 @@ Tipi.serve('0.0.0.0', 1234, opts) do |req|
     req.send_chunk("foo\n")
     sleep 0.5
     req.send_chunk("bar\n", done: true)
+  elsif req.path == '/upload'
+    body = req.read
+    req.respond("Body: #{body.inspect} (#{body.bytesize} bytes)")
   else
     req.respond("Hello world!\n")
   end

data/examples/servername_cb.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require 'openssl'
+require 'fiber'
+
+ctx = OpenSSL::SSL::SSLContext.new
+
+f = Fiber.new { |peer| loop { p peer: peer; _name, peer = peer.transfer nil } }
+ctx.servername_cb = proc { |_socket, name|
+  p servername_cb: name
+  f.transfer([name, Fiber.current]).tap { |r| p result: r }
+}
+
+socket = Socket.new(:INET, :STREAM).tap do |s|
+  s.setsockopt(Socket::SOL_SOCKET, Socket::SO_REUSEADDR, 1)
+  s.setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_REUSEPORT, 1)
+  s.bind(Socket.sockaddr_in(12345, '0.0.0.0'))
+  s.listen(Socket::SOMAXCONN)
+end
+server = OpenSSL::SSL::SSLServer.new(socket, ctx)
+
+Thread.new do
+  sleep 0.5
+  socket = TCPSocket.new('127.0.0.1', 12345)
+  client = OpenSSL::SSL::SSLSocket.new(socket)
+  client.hostname = 'example.com'
+  p client: client
+  client.connect
+rescue => e
+  p client_error: e
+end
+
+while true
+  conn = server.accept
+  p accepted: conn
+  break
+end

data/examples/websocket_demo.rb

@@ -1,13 +1,7 @@
 # frozen_string_literal: true
 
-require 'bundler/inline'
-
-gemfile do
-  source 'https://rubygems.org'
-  gem 'polyphony', '~> 0.44'
-  gem 'tipi', '~> 0.31'
-end
-
+require 'bundler/setup'
+require 'tipi'
 require 'tipi/websocket'
 
 class WebsocketClient

data/examples/ws_page.html

@@ -6,7 +6,7 @@
 <body>
   <script>
     var connect = function () {
-      var exampleSocket = new WebSocket("wss://dev.realiteq.net/");
+      var exampleSocket = new WebSocket("ws://localhost:4411/");
 
       exampleSocket.onopen = function (event) {
         document.querySelector('#status').innerText = 'connected';
@@ -30,4 +30,4 @@
   <h1 id="status">disconnected</h1>
   <h1 id="msg"></h1>
 </body>
-</html>
+</html>

data/lib/tipi.rb

@@ -1,9 +1,18 @@
 # frozen_string_literal: true
 
 require 'polyphony'
+
 require_relative './tipi/http1_adapter'
 require_relative './tipi/http2_adapter'
 require_relative './tipi/configuration'
+require_relative './tipi/response_extensions'
+require_relative './tipi/acme'
+
+require 'qeweney/request'
+
+class Qeweney::Request
+  include Tipi::ResponseExtensions
+end
 
 module Tipi
   ALPN_PROTOCOLS = %w[h2 http/1.1].freeze
@@ -42,7 +51,7 @@ module Tipi
     ensure
       client.close rescue nil
     end
-    
+
     def protocol_adapter(socket, opts)
       use_http2 = socket.respond_to?(:alpn_protocol) &&
                   socket.alpn_protocol == H2_PROTOCOL
@@ -53,5 +62,84 @@ module Tipi
     def route(&block)
       proc { |req| req.route(&block) }
     end
+
+    CERTIFICATE_STORE_DEFAULT_DIR = File.expand_path('~/.tipi')
+    CERTIFICATE_STORE_DEFAULT_DB_PATH = File.join(
+      CERTIFICATE_STORE_DEFAULT_DIR, 'certificates.db'
+    )
+
+    def default_certificate_store
+      FileUtils.mkdir(CERTIFICATE_STORE_DEFAULT_DIR) rescue nil
+      Tipi::ACME::SQLiteCertificateStore.new(CERTIFICATE_STORE_DEFAULT_DB_PATH)
+    end
+
+    def full_service(
+      http_port: 10080,
+      https_port: 10443,
+      certificate_store: default_certificate_store,
+      app: nil, &block
+    )
+      app ||= block
+      raise "No app given" unless app
+
+      http_handler = ->(r) { r.redirect("https://#{r.host}#{r.path}") }
+    
+      ctx = OpenSSL::SSL::SSLContext.new
+      # ctx.ciphers = 'ECDH+aRSA'
+      Polyphony::Net.setup_alpn(ctx, Tipi::ALPN_PROTOCOLS)
+    
+      challenge_handler = Tipi::ACME::HTTPChallengeHandler.new
+      certificate_manager = Tipi::ACME::CertificateManager.new(
+        master_ctx: ctx,
+        store: certificate_store,
+        challenge_handler: challenge_handler
+      )
+    
+      http_listener = spin do
+        opts = {
+          reuse_addr:   true,
+          reuse_port:   true,
+          dont_linger:  true,
+        }
+        puts "Listening for HTTP on localhost:#{http_port}"
+        server = Polyphony::Net.tcp_listen('0.0.0.0', http_port, opts)
+        wrapped_handler = certificate_manager.challenge_routing_app(http_handler)
+        server.accept_loop do |client|
+          spin do
+            Tipi.client_loop(client, opts, &wrapped_handler)
+          rescue => e
+            puts "Uncaught error in HTTP listener: #{e.inspect}"
+          end
+        end
+      ensure
+        server.close
+      end
+    
+      https_listener = spin do
+        opts = {
+          reuse_addr:     true,
+          reuse_port:     true,
+          dont_linger:    true,
+          secure_context: ctx,
+        }
+      
+        puts "Listening for HTTPS on localhost:#{https_port}"
+        server = Polyphony::Net.tcp_listen('0.0.0.0', https_port, opts)
+        loop do
+          client = server.accept
+          spin do
+            Tipi.client_loop(client, opts, &app)
+          rescue => e
+            puts "Uncaught error in HTTPS listener: #{e.inspect}"
+          end
+        rescue OpenSSL::SSL::SSLError, SystemCallError, TypeError
+          # ignore
+        end
+      ensure
+        server.close
+      end
+
+      Fiber.await(http_listener, https_listener)
+    end
   end
 end