tipi 0.42 → 0.43

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 82155f4b86223d3fb32dc8c314abfeb75fda951747e63832ed5c15bc3b1f43cc
-  data.tar.gz: 721b98fb8d330d1bc38f8f236f82ddd909235d31f5875c1b4f89895e4e3926da
+  metadata.gz: c7a42c93f436a24b74aef2fc9de54d7f01b587ec42b2897d0b0c24a9090776d8
+  data.tar.gz: 1fa532504e9c79d620306b4ddfbd01bf1d1dc39b563ec2a2c3cfec9cb00a1b64
 SHA512:
-  metadata.gz: 10389779975234d90c968626b4f9d6b168491f9a32ecf1be71683c37cf11f58f1bac52a36478691e72328236a16d309a8462104bda556772e25cebe52b2b0d53
-  data.tar.gz: 99db2c15278cdfc286ffad05f266f5d62eec6a3bb1363f8dfa4c302549a308a691496ee60ec085ede59e3b594533e8e196bbca144878e5a6dccf2ecfc962151c
+  metadata.gz: 93282cde73eca46289cf212e76c90e7390dcd9e48e8594bd756d5f59ae3bc5fed41e61d216c85cca9c9d2c8fca1061ee0159f8cc0ca575fe0b7149a21b1edc24
+  data.tar.gz: 6fd1dc4de05351d0b7da91a7344798145f31659f4663f451355222ce63bb8504fb123c562199820b4f217bc14d064bb529fcc0c9ea495863856bd79f073a491b

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.43 2021-08-20
+
+- Extract HTTP/1 parser into a separate gem:
+  [H1P](https://github.com/digital-fabric/h1p)
+
 ## 0.42 2021-08-16
 
 - HTTP/1 parser:  disable UTF-8 parsing for all but header values

data/Gemfile

@@ -1,7 +1,7 @@
 source 'https://rubygems.org'
 
 gemspec
-%w{polyphony qeweney}.each do |dep|
+%w{polyphony qeweney h1p}.each do |dep|
   dir = "../#{dep}"
   gem(dep, path: dir) if File.directory?(dir)
 end

data/Gemfile.lock

@@ -1,7 +1,12 @@
+PATH
+  remote: ../h1p
+  specs:
+    h1p (0.2)
+
 PATH
   remote: ../polyphony
   specs:
-    polyphony (0.69)
+    polyphony (0.70)
 
 PATH
   remote: ../qeweney
@@ -12,9 +17,10 @@ PATH
 PATH
   remote: .
   specs:
-    tipi (0.42)
+    tipi (0.43)
       acme-client (~> 2.0.8)
       extralite (~> 1.2)
+      h1p (~> 0.2)
       http-2 (~> 0.11)
       localhost (~> 1.1.4)
       msgpack (~> 1.4.2)
@@ -28,13 +34,11 @@ GEM
   specs:
     acme-client (2.0.8)
       faraday (>= 0.17, < 2.0.0)
-    ansi (1.5.0)
-    builder (3.2.4)
     cuba (3.9.3)
       rack (>= 1.6.0)
     docile (1.4.0)
     escape_utils (1.2.1)
-    extralite (1.2)
+    extralite (1.3)
     faraday (1.7.0)
       faraday-em_http (~> 1.0)
       faraday-em_synchrony (~> 1.0)
@@ -55,22 +59,14 @@ GEM
     faraday-patron (1.0.0)
     faraday-rack (1.0.0)
     http-2 (0.11.0)
-    http_parser.rb (0.7.0)
     json (2.5.1)
     localhost (1.1.8)
+    memory_profiler (1.0.0)
     minitest (5.11.3)
-    minitest-reporters (1.4.3)
-      ansi
-      builder
-      minitest (>= 5.0)
-      ruby-progressbar
     msgpack (1.4.2)
     multipart-post (2.1.1)
     rack (2.2.3)
-    rake (12.3.3)
-    rake-compiler (1.1.1)
-      rake
-    ruby-progressbar (1.11.0)
+    rake (13.0.6)
     ruby2_keywords (0.0.5)
     simplecov (0.17.1)
       docile (~> 1.1)
@@ -84,13 +80,12 @@ PLATFORMS
 
 DEPENDENCIES
   cuba (~> 3.9.3)
-  http_parser.rb (= 0.7.0)
+  h1p!
+  memory_profiler (~> 1.0.0)
   minitest (~> 5.11.3)
-  minitest-reporters (~> 1.4.2)
   polyphony!
   qeweney!
-  rake (~> 12.3.3)
-  rake-compiler (= 1.1.1)
+  rake (~> 13.0.6)
   simplecov (~> 0.17.1)
   tipi!
 

data/benchmarks/bm_http1_parser.rb

@@ -4,6 +4,25 @@ require 'bundler/setup'
 
 HTTP_REQUEST = "GET /foo HTTP/1.1\r\nHost: example.com\r\nAccept: */*\r\n\r\n"
 
+def measure_time_and_allocs
+  4.times { GC.start }
+  GC.disable
+
+  t0 = Time.now
+  a0 = object_count
+  yield
+  t1 = Time.now
+  a1 = object_count
+  [t1 - t0, a1 - a0]
+ensure
+  GC.enable
+end
+
+def object_count
+  count = ObjectSpace.count_objects
+  count[:TOTAL] - count[:FREE]
+end
+
 def benchmark_other_http1_parser(iterations)
   STDOUT << "http_parser.rb: "
   require 'http_parser.rb'
@@ -16,21 +35,20 @@ def benchmark_other_http1_parser(iterations)
     headers = h
     headers[':method'] = parser.http_method
     headers[':path'] = parser.request_url
-    headers[':protocol'] = parser.http_version
   end
   parser.on_message_complete = proc { done = true }
 
-  t0 = Time.now
-  iterations.times do
-    o << HTTP_REQUEST
-    done = false
-    while !done
-      msg = i.readpartial(4096)
-      parser << msg
+  elapsed, allocated = measure_time_and_allocs do
+    iterations.times do
+      o << HTTP_REQUEST
+      done = false
+      while !done
+        msg = i.readpartial(4096)
+        parser << msg
+      end
     end
   end
-  t1 = Time.now
-  puts "#{iterations / (t1 - t0)} ips"
+  puts(format('elapsed: %f, allocated: %d (%f/req), rate: %f ips', elapsed, allocated, allocated.to_f / iterations, iterations / elapsed))
 end
 
 def benchmark_tipi_http1_parser(iterations)
@@ -40,22 +58,28 @@ def benchmark_tipi_http1_parser(iterations)
   reader = proc { |len| i.readpartial(len) }
   parser = Tipi::HTTP1Parser.new(reader)
 
-  t0 = Time.now
-  iterations.times do
-    o << HTTP_REQUEST
-    headers = parser.parse_headers
+  elapsed, allocated = measure_time_and_allocs do
+    iterations.times do
+      o << HTTP_REQUEST
+      headers = parser.parse_headers
+    end
   end
-  t1 = Time.now
-  puts "#{iterations / (t1 - t0)} ips"
+  puts(format('elapsed: %f, allocated: %d (%f/req), rate: %f ips', elapsed, allocated, allocated.to_f / iterations, iterations / elapsed))
 end
 
 def fork_benchmark(method, iterations)
-  pid = fork { send(method, iterations) }
+  pid = fork do
+    send(method, iterations)
+  rescue Exception => e
+    p e
+    p e.backtrace
+    exit!
+  end
   Process.wait(pid)
 end
 
 x = 500000
-fork_benchmark(:benchmark_other_http1_parser, x)
-fork_benchmark(:benchmark_tipi_http1_parser, x)
+# fork_benchmark(:benchmark_other_http1_parser, x)
+# fork_benchmark(:benchmark_tipi_http1_parser, x)
 
-# benchmark_tipi_http1_parser(x)
+benchmark_tipi_http1_parser(x)

data/df/server.rb

@@ -8,6 +8,8 @@ listeners = [
   listen_unix
 ]
 
+spin_loop(interval: 60) { GC.compact } if GC.respond_to?(:compact)
+
 begin
   log('Starting DF server')
   Fiber.await(*listeners)

data/df/server_utils.rb

@@ -64,17 +64,14 @@ def listen_https
     c = IO.read('../../reality/ssl/cacert.pem')
     certificates = c.scan(CERTIFICATE_REGEXP).map { |p|  OpenSSL::X509::Certificate.new(p.first) }
     ctx = OpenSSL::SSL::SSLContext.new
+    ctx.security_level = 0
     cert = certificates.shift
     log "SSL Certificate expires: #{cert.not_after.inspect}"
     ctx.add_certificate(cert, private_key, certificates)
     ctx.ciphers = 'ECDH+aRSA'
-    ctx.send(
-      :set_minmax_proto_version,
-      OpenSSL::SSL::SSL3_VERSION,
-      OpenSSL::SSL::TLS1_3_VERSION
-    )
-    # ctx.min_version = OpenSSL::SSL::SSL3_VERSION #OpenSSL::SSL::TLS1_VERSION
-    # ctx.max_version = OpenSSL::SSL::TLS1_3_VERSION
+    ctx.max_version = OpenSSL::SSL::TLS1_3_VERSION
+    ctx.min_version = OpenSSL::SSL::SSL3_VERSION
+    ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE
 
     # TODO: further limit ciphers
     # ref: https://github.com/socketry/falcon/blob/3ec805b3ceda0a764a2c5eb68cde33897b6a35ff/lib/falcon/environments/tls.rb
@@ -91,10 +88,10 @@ def listen_https
     server = Polyphony::Net.tcp_listen('0.0.0.0', 10443, opts)
     id = 0
     loop do
-      log('Before HTTPS server.accept')
-      client = server.accept
-      log('After HTTPS server.accept')
-      log('Accept HTTPS client connection', client: client)
+      client = server.accept rescue nil
+      next unless client
+
+      # log('Accept HTTPS client connection', client: client)
       spin("https#{id += 1}") do
         @service.incr_connection_count
         Tipi.client_loop(client, opts) { |req| @service.http_request(req) }
@@ -104,12 +101,12 @@ def listen_https
         # log("Done with HTTP connection", client: client)
         @service.decr_connection_count
       end
-    rescue OpenSSL::SSL::SSLError, SystemCallError, TypeError => e
-      log('HTTPS accept error', error: e)
+    # rescue OpenSSL::SSL::SSLError, SystemCallError, TypeError => e
+    #   log('HTTPS accept error', error: e)
     rescue Polyphony::BaseException
       raise
     rescue Exception => e
-      log 'HTTPS accept (unknown) error', error: e, backtrace: e.backtrace
+      log 'HTTPS listener error: ', error: e, backtrace: e.backtrace
     end
   end
 end

data/lib/tipi/digital_fabric/service.rb

@@ -50,6 +50,8 @@ module DigitalFabric
       switch_rate = backend_stats[:switch_count] / elapsed
       poll_rate = backend_stats[:poll_count] / elapsed
 
+      object_space_stats = ObjectSpace.count_objects
+
       {
         service: {
           agent_count: @agents.size,
@@ -73,6 +75,8 @@ module DigitalFabric
         process: {
           cpu_usage: cpu,
           rss: rss.to_f / 1024,
+          objects_total: object_space_stats[:TOTAL],
+          objects_free:  object_space_stats[:FREE]
         }
       }
     end

data/lib/tipi/http1_adapter.rb

@@ -1,9 +1,10 @@
 # frozen_string_literal: true
 
-require 'tipi_ext'
-require_relative './http2_adapter'
+require 'h1p'
 require 'qeweney/request'
 
+require_relative './http2_adapter'
+
 module Tipi
   # HTTP1 protocol implementation
   class HTTP1Adapter
@@ -14,7 +15,7 @@ module Tipi
       @conn = conn
       @opts = opts
       @first = true
-      @parser = Tipi::HTTP1Parser.new(@conn)
+      @parser = H1P::Parser.new(@conn)
     end
     
     def each(&block)
@@ -26,7 +27,7 @@ module Tipi
         # upgraded
         break if handle_request(headers, &block)
       end
-    rescue Tipi::HTTP1Parser::Error
+    rescue H1P::Error
       # ignore
     rescue SystemCallError, IOError
       # ignore

data/lib/tipi/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Tipi
-  VERSION = '0.42'
+  VERSION = '0.43'
 end

data/test/helper.rb

@@ -8,7 +8,6 @@ require_relative './eg'
 require_relative './coverage' if ENV['COVERAGE']
 
 require 'minitest/autorun'
-require 'minitest/reporters'
 
 require 'polyphony'
 

data/test/test_request.rb

@@ -39,7 +39,7 @@ class RequestHeadersTest < MiniTest::Test
     assert_equal 'blah.com', req.headers['host']
     assert_equal 'bar', req.headers['foo']
     assert_equal ['1', '2', '3'], req.headers['hi']
-    assert_equal 'get', req.headers[':method']
+    assert_equal 'GET', req.headers[':method']
     assert_equal '/titi', req.headers[':path']
   end
 

data/tipi.gemspec

@@ -14,7 +14,6 @@ Gem::Specification.new do |s|
   }
   s.rdoc_options = ["--title", "tipi", "--main", "README.md"]
   s.extra_rdoc_files = ["README.md"]
-  s.extensions = ["ext/tipi/extconf.rb"]
   s.require_paths = ["lib"]
   s.required_ruby_version = '>= 2.6'
 
@@ -23,6 +22,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency      'polyphony',          '~>0.69'
   s.add_runtime_dependency      'qeweney',            '~>0.14'
   s.add_runtime_dependency      'extralite',          '~>1.2'
+  s.add_runtime_dependency      'h1p',                '~>0.2'
   
   s.add_runtime_dependency      'http-2',             '~>0.11'
   s.add_runtime_dependency      'rack',               '>=2.0.8', '<2.3.0'
@@ -33,12 +33,10 @@ Gem::Specification.new do |s|
   # for digital fabric
   s.add_runtime_dependency      'msgpack',            '~>1.4.2'
 
-  s.add_development_dependency  'rake-compiler',        '1.1.1'
-  s.add_development_dependency  'rake',               '~>12.3.3'
+  s.add_development_dependency  'rake',               '~>13.0.6'
   s.add_development_dependency  'minitest',           '~>5.11.3'
-  s.add_development_dependency  'minitest-reporters', '~>1.4.2'
   s.add_development_dependency  'simplecov',          '~>0.17.1'
+  s.add_development_dependency  'memory_profiler',    '~>1.0.0'
 
   s.add_development_dependency  'cuba',               '~>3.9.3'
-  s.add_development_dependency  'http_parser.rb',     '0.7.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tipi
 version: !ruby/object:Gem::Version
-  version: '0.42'
+  version: '0.43'
 platform: ruby
 authors:
 - Sharon Rosner
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-08-16 00:00:00.000000000 Z
+date: 2021-08-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: polyphony
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: h1p
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
 - !ruby/object:Gem::Dependency
   name: http-2
   requirement: !ruby/object:Gem::Requirement
@@ -142,34 +156,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.4.2
-- !ruby/object:Gem::Dependency
-  name: rake-compiler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 1.1.1
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 1.1.1
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 12.3.3
+        version: 13.0.6
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 12.3.3
+        version: 13.0.6
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -185,33 +185,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 5.11.3
 - !ruby/object:Gem::Dependency
-  name: minitest-reporters
+  name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.4.2
+        version: 0.17.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.4.2
+        version: 0.17.1
 - !ruby/object:Gem::Dependency
-  name: simplecov
+  name: memory_profiler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.17.1
+        version: 1.0.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.17.1
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: cuba
   requirement: !ruby/object:Gem::Requirement
@@ -226,26 +226,11 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 3.9.3
-- !ruby/object:Gem::Dependency
-  name: http_parser.rb
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 0.7.0
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 0.7.0
 description: 
 email: sharon@noteflakes.com
 executables:
 - tipi
-extensions:
-- ext/tipi/extconf.rb
+extensions: []
 extra_rdoc_files:
 - README.md
 files:
@@ -308,10 +293,6 @@ files:
 - examples/websocket_server.rb
 - examples/ws_page.html
 - examples/wss_page.html
-- ext/tipi/extconf.rb
-- ext/tipi/http1_parser.c
-- ext/tipi/http1_parser.h
-- ext/tipi/tipi_ext.c
 - lib/tipi.rb
 - lib/tipi/acme.rb
 - lib/tipi/cli.rb
@@ -333,12 +314,10 @@ files:
 - lib/tipi/response_extensions.rb
 - lib/tipi/version.rb
 - lib/tipi/websocket.rb
-- security/http1.rb
 - test/coverage.rb
 - test/eg.rb
 - test/helper.rb
 - test/run.rb
-- test/test_http1_parser.rb
 - test/test_http_server.rb
 - test/test_request.rb
 - tipi.gemspec
@@ -366,7 +345,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Tipi - the All-in-one Web Server for Ruby Apps