itsi 0.2.26 → 0.2.27.rc1

data/gems/server/Gemfile.lock

@@ -0,0 +1,123 @@
+PATH
+  remote: ../scheduler
+  specs:
+    itsi-scheduler (0.2.27.rc1)
+      rb_sys (~> 0.9.91)
+
+PATH
+  remote: .
+  specs:
+    itsi-server (0.2.27.rc1)
+      json (~> 2)
+      prism (~> 1.4)
+      rack (>= 1.6)
+      rb_sys (~> 0.9.91)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ansi (1.6.0)
+    base64 (0.3.0)
+    bigdecimal (4.1.2)
+    builder (3.3.0)
+    connection_pool (3.0.2)
+    google-protobuf (4.30.2)
+      bigdecimal
+      rake (>= 13)
+    googleapis-common-protos-types (1.19.0)
+      google-protobuf (>= 3.18, < 5.a)
+    grpc (1.78.0)
+      google-protobuf (>= 3.25, < 5.0)
+      googleapis-common-protos-types (~> 1.0)
+    json (2.19.4)
+    jwt (3.1.2)
+      base64
+    language_server-protocol (3.17.0.5)
+    logger (1.7.0)
+    minitest (5.27.0)
+    minitest-reporters (1.8.0)
+      ansi
+      builder
+      minitest (>= 5.0, < 7)
+      ruby-progressbar
+    net_http_unix (0.2.2)
+    prism (1.9.0)
+    rack (3.2.6)
+    rackup (2.3.1)
+      rack (>= 3)
+    rake (13.3.1)
+    rake-compiler (1.3.0)
+      rake
+    rake-compiler-dock (1.11.0)
+    rb_sys (0.9.126)
+      json (>= 2)
+      rake-compiler-dock (= 1.11.0)
+    rbs (3.10.3)
+      logger
+      tsort
+    redis (5.4.1)
+      redis-client (>= 0.22.0)
+    redis-client (0.24.0)
+      connection_pool
+    ruby-lsp (0.26.7)
+      language_server-protocol (~> 3.17.0)
+      prism (>= 1.2, < 2.0)
+      rbs (>= 3, < 5)
+    ruby-progressbar (1.13.0)
+    tsort (0.2.0)
+
+PLATFORMS
+  arm64-darwin-24
+  ruby
+
+DEPENDENCIES
+  bundler
+  grpc
+  itsi-scheduler!
+  itsi-server!
+  jwt
+  minitest (~> 5.16)
+  minitest-reporters
+  net_http_unix
+  rack
+  rackup
+  rake (~> 13.0)
+  rake-compiler
+  rb_sys (~> 0.9.91)
+  redis
+  ruby-lsp
+
+CHECKSUMS
+  ansi (1.6.0) sha256=ac9ea0c0ea8d32fb4e271348e609963ac78882f34b73836c2a02b3622e666658
+  base64 (0.3.0) sha256=27337aeabad6ffae05c265c450490628ef3ebd4b67be58257393227588f5a97b
+  bigdecimal (4.1.2) sha256=53d217666027eab4280346fba98e7d5b66baaae1b9c3c1c0ffe89d48188a3fbd
+  builder (3.3.0) sha256=497918d2f9dca528fdca4b88d84e4ef4387256d984b8154e9d5d3fe5a9c8835f
+  connection_pool (3.0.2) sha256=33fff5ba71a12d2aa26cb72b1db8bba2a1a01823559fb01d29eb74c286e62e0a
+  google-protobuf (4.30.2) sha256=0f35168dbeeccf13d928acf6c128cfec17b9a826ae4505246a02c115f4ae16ed
+  googleapis-common-protos-types (1.19.0) sha256=aecb76ca5326f8bcc47ab083259bbc4971d07e87f56808af7e210669d9765694
+  grpc (1.78.0) sha256=7aaf47b6d7783fb0e7d40fc853d34e802d47aef7b1312862b6e719141b3527c9
+  itsi-scheduler (0.2.27.rc1)
+  itsi-server (0.2.27.rc1)
+  json (2.19.4) sha256=670a7d333fb3b18ca5b29cb255eb7bef099e40d88c02c80bd42a3f30fe5239ac
+  jwt (3.1.2) sha256=af6991f19a6bb4060d618d9add7a66f0eeb005ac0bc017cd01f63b42e122d535
+  language_server-protocol (3.17.0.5) sha256=fd1e39a51a28bf3eec959379985a72e296e9f9acfce46f6a79d31ca8760803cc
+  logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203
+  minitest (5.27.0) sha256=2d3b17f8a36fe7801c1adcffdbc38233b938eb0b4966e97a6739055a45fa77d5
+  minitest-reporters (1.8.0) sha256=8ce5280fb73ad3178ae525454df169b6f28c1b38b1d088ea91815d3a370ba384
+  net_http_unix (0.2.2) sha256=98aa0e1e7787f8383e8dd8ff60fc18062c2ddb2aadbc76e92cfb4f95d2c9d71d
+  prism (1.9.0) sha256=7b530c6a9f92c24300014919c9dcbc055bf4cdf51ec30aed099b06cd6674ef85
+  rack (3.2.6) sha256=5ed78e1f73b2e25679bec7d45ee2d4483cc4146eb1be0264fc4d94cb5ef212c2
+  rackup (2.3.1) sha256=6c79c26753778e90983761d677a48937ee3192b3ffef6bc963c0950f94688868
+  rake (13.3.1) sha256=8c9e89d09f66a26a01264e7e3480ec0607f0c497a861ef16063604b1b08eb19c
+  rake-compiler (1.3.0) sha256=eec272ef6d4dad27b36f5cdcf5b9ee4df2193751f4082b095f981ebf9cdf4127
+  rake-compiler-dock (1.11.0) sha256=eab51f2cd533eb35cea6b624a75281f047123e70a64c58b607471bb49428f8c2
+  rb_sys (0.9.126) sha256=ba958e0b8b4b89eeae0b3d24b64c809eb2c37e0ab0773a49e9b1c2e22c95aef8
+  rbs (3.10.3) sha256=70627f3919016134d554e6c99195552ae3ef6020fe034c8e983facc9c192daa6
+  redis (5.4.1) sha256=b5e675b57ad22b15c9bcc765d5ac26f60b675408af916d31527af9bd5a81faae
+  redis-client (0.24.0) sha256=ee65ee39cb2c38608b734566167fd912384f3c1241f59075e22858f23a085dbb
+  ruby-lsp (0.26.7) sha256=60a1199fc7e329348d63a2479854f94435725d833eeabf3d539b790185cbf21f
+  ruby-progressbar (1.13.0) sha256=80fc9c47a9b640d6834e0dc7b3c94c9df37f08cb072b7761e4a71e22cff29b33
+  tsort (0.2.0) sha256=9650a793f6859a43b6641671278f79cfead60ac714148aabe4e3f0060480089f
+
+BUNDLED WITH
+  2.6.9

data/gems/server/Rakefile

@@ -29,6 +29,16 @@ SMOKE_TEST_GLOBS = %w[
   test/middleware/string_rewrite.rb
 ].freeze
 
+def native_build_tasks_requested?
+  requested = Rake.application.top_level_tasks
+  return true if requested.empty?
+
+  requested.any? do |task_name|
+    task_name == "default" ||
+      task_name.start_with?("build", "compile", "cross", "clobber")
+  end
+end
+
 def configure_test_task(task_name, test_globs)
   Minitest::TestTask.create(task_name) do |t|
     t.libs << "test"
@@ -41,17 +51,20 @@ end
 
 configure_test_task(:test, ["test/**/*.rb"])
 configure_test_task("test:smoke", SMOKE_TEST_GLOBS)
+configure_test_task("test:acme", ["test/acme/**/*.rb"])
 
 task "test:full" => :test
 
-require "rb_sys/extensiontask"
+if native_build_tasks_requested?
+  require "rb_sys/extensiontask"
 
-task build: :compile
+  task build: :compile
 
-GEMSPEC = Gem::Specification.load("itsi-server.gemspec")
+  GEMSPEC = Gem::Specification.load("itsi-server.gemspec")
 
-RbSys::ExtensionTask.new("itsi-server", GEMSPEC) do |ext|
-  ext.lib_dir = "lib/itsi/server"
+  RbSys::ExtensionTask.new("itsi-server", GEMSPEC) do |ext|
+    ext.lib_dir = "lib/itsi/server"
+  end
 end
 
 task default: %i[compile test rubocop]

data/gems/server/lib/itsi/http_request.rb

@@ -145,6 +145,16 @@ module Itsi
       end
     end
 
+    def partial_hijack
+      UNIXSocket.pair.yield_self do |(server_sock, app_sock)|
+        server_sock.autoclose = false
+        response.partial_hijack(server_sock.fileno)
+        server_sock.sync = true
+        app_sock.sync = true
+        app_sock
+      end
+    end
+
     # Rack expects env["rack.hijack"] to respond to #call.
     def call
       hijack

data/gems/server/lib/itsi/server/rack_interface.rb

@@ -1,6 +1,45 @@
 module Itsi
   class Server
     module RackInterface
+      class PartialHijackStream
+        def initialize(response)
+          @response = response
+        end
+
+        def write(chunk)
+          @response.write(chunk.to_s)
+        end
+
+        def read(*)
+          nil
+        end
+
+        def <<(chunk)
+          write(chunk)
+          self
+        end
+
+        def flush
+          self
+        end
+
+        def close_write
+          @response.close_write
+        end
+
+        def close_read
+          true
+        end
+
+        def close
+          @response.close_write
+        end
+
+        def closed?
+          @response.closed?
+        end
+      end
+
       # Builds a handler proc that is compatible with Rack applications.
       def self.for(app)
         require "rack"
@@ -38,7 +77,8 @@ module Itsi
         response.status = status
 
         # 2. Set Headers
-        body_streamer = streaming_body?(body) ? body : headers.delete("rack.hijack")
+        hijack_callback = headers.delete("rack.hijack")
+        body_streamer = streaming_body?(body) ? body : hijack_callback
 
         response.reserve_headers(headers.size)
 
@@ -57,7 +97,10 @@ module Itsi
         # the server will begin to stream it to the client.
 
 
-        if body_streamer
+        if hijack_callback
+          stream = status == 101 ? request.partial_hijack : PartialHijackStream.new(response)
+          body_streamer.call(stream)
+        elsif body_streamer
           # If we're partially hijacked or returned a streaming body,
           # stream this response.
           body_streamer.call(response)

data/gems/server/lib/itsi/server/version.rb

@@ -2,6 +2,6 @@
 
 module Itsi
   class Server
-    VERSION = "0.2.26"
+    VERSION = "0.2.27.rc1"
   end
 end

data/gems/server/lib/itsi/server.rb

@@ -32,6 +32,30 @@ module Itsi
         @running && !@running.empty?
       end
 
+      def current_server
+        @running&.last || raise("No running Itsi::Server instance")
+      end
+
+      def tls_bindings
+        current_server.tls_bindings
+      end
+
+      def tls_domains(listener_id = nil)
+        current_server.tls_domains(listener_id)
+      end
+
+      def tls_domain_statuses(listener_id = nil)
+        current_server.tls_domain_statuses(listener_id)
+      end
+
+      def register_tls_domain(domain, listener_id = nil)
+        current_server.register_tls_domain(domain, listener_id)
+      end
+
+      def unregister_tls_domain(domain, listener_id = nil)
+        current_server.unregister_tls_domain(domain, listener_id)
+      end
+
       def start_in_background_thread(cli_params = {}, &blk)
         @background_threads ||= []
         server, background_thread = start(cli_params, background: true, &blk)

data/gems/server/test/acme/local_acme_challenges.rb

@@ -0,0 +1,190 @@
+# frozen_string_literal: true
+
+require "helpers/test_helper"
+require "helpers/local_acme"
+require "socket"
+require "timeout"
+
+class TestLocalAcmeChallenges < Minitest::Test
+  APP = proc { |_env| [200, { "content-type" => "text/plain" }, ["acme-ok"]] }
+
+  class << self
+    def local_acme
+      raise Minitest::Skip, "Go is required for local ACME tests" unless LocalAcmeAuthority.available?
+
+      return @local_acme if @local_acme
+
+      @local_acme = LocalAcmeAuthority.new
+      @local_acme.start
+      @previous_env = {}
+      @local_acme.env.each do |key, value|
+        @previous_env[key] = ENV[key]
+        ENV[key] = value
+      end
+      @local_acme
+    end
+
+    def shutdown
+      return unless @local_acme
+
+      @previous_env&.each do |key, value|
+        value.nil? ? ENV.delete(key) : ENV[key] = value
+      end
+      @local_acme.stop
+    end
+  end
+
+  def test_tls_alpn01_issuance_without_http_listener
+    domain = "alpn.itsi.test"
+    local_acme = self.class.local_acme
+    https_bind = "https://0.0.0.0:#{local_acme.tls_port}?cert=acme&domains=#{domain}&acme_email=test@example.com"
+
+    server(app: APP, bind: https_bind) do
+      wait_for_https_response(local_acme.tls_port, domain)
+
+      certificate = peer_certificate(local_acme.tls_port, domain)
+      assert_certificate_domain(certificate, domain)
+
+      response = https_get(local_acme.tls_port, "/", domain)
+      assert_equal "200", response.code
+      assert_equal "acme-ok", response.body
+    end
+  end
+
+  def test_http01_issuance_when_tls_validation_port_is_unavailable
+    domain = "http01.itsi.test"
+    local_acme = self.class.local_acme
+    app_port = free_tcp_port
+    https_bind = "https://0.0.0.0:#{app_port}?cert=acme&domains=#{domain}&acme_email=test@example.com"
+    http_bind = "http://0.0.0.0:#{local_acme.http_port}"
+
+    server(app: APP, bind: https_bind, binds: [https_bind, http_bind]) do
+      wait_for_https_response(app_port, domain)
+
+      certificate = peer_certificate(app_port, domain)
+      assert_certificate_domain(certificate, domain)
+
+      response = https_get(app_port, "/", domain)
+      assert_equal "200", response.code
+      assert_equal "acme-ok", response.body
+    end
+  end
+
+  def test_dynamic_http01_issuance_with_runtime_domain_registration
+    domain = "runtime-http01.itsi.test"
+    local_acme = self.class.local_acme
+    app_port = free_tcp_port
+    https_bind = "https://0.0.0.0:#{app_port}?cert=acme&acme_email=test@example.com"
+    http_bind = "http://0.0.0.0:#{local_acme.http_port}"
+
+    server(app: APP, bind: https_bind, binds: [https_bind, http_bind]) do
+      assert_equal [], Itsi::Server.tls_domains
+      assert_equal ["tcp://0.0.0.0:#{app_port}"], Itsi::Server.tls_bindings
+
+      Itsi::Server.register_tls_domain(domain)
+      wait_until { Itsi::Server.tls_domains.include?(domain) }
+      wait_for_https_response(app_port, domain)
+
+      statuses = Itsi::Server.tls_domain_statuses
+      active = statuses.find { |status| status["domain"] == domain }
+      refute_nil active
+      assert_equal "active", active["status"]
+
+      certificate = peer_certificate(app_port, domain)
+      assert_certificate_domain(certificate, domain)
+
+      Itsi::Server.unregister_tls_domain(domain)
+      wait_until { !Itsi::Server.tls_domains.include?(domain) }
+    end
+  end
+
+  def test_dynamic_tls_alpn01_issuance_with_runtime_domain_registration
+    domain = "runtime-alpn.itsi.test"
+    local_acme = self.class.local_acme
+    https_bind = "https://0.0.0.0:#{local_acme.tls_port}?cert=acme&acme_email=test@example.com"
+
+    server(app: APP, bind: https_bind) do
+      assert_equal [], Itsi::Server.tls_domains
+
+      Itsi::Server.register_tls_domain(domain)
+      wait_until { Itsi::Server.tls_domains.include?(domain) }
+      wait_for_https_response(local_acme.tls_port, domain)
+
+      statuses = Itsi::Server.tls_domain_statuses
+      active = statuses.find { |status| status["domain"] == domain }
+      refute_nil active
+      assert_equal "active", active["status"]
+
+      certificate = peer_certificate(local_acme.tls_port, domain)
+      assert_certificate_domain(certificate, domain)
+    end
+  end
+
+  private
+
+  def free_tcp_port
+    server = TCPServer.new("127.0.0.1", 0)
+    port = server.addr[1]
+    server.close
+    port
+  end
+
+  def wait_for_https_response(port, host = "127.0.0.1")
+    Timeout.timeout(20) do
+      loop do
+        begin
+          response = https_get(port, "/", host)
+          return response if response.code == "200"
+        rescue StandardError
+          sleep 0.1
+        end
+      end
+    end
+  end
+
+  def wait_until(timeout: 10)
+    Timeout.timeout(timeout) do
+      loop do
+        return if yield
+
+        sleep 0.05
+      end
+    end
+  end
+
+  def https_get(port, path, host = "127.0.0.1")
+    Net::HTTP.start(
+      "127.0.0.1",
+      port,
+      use_ssl: true,
+      verify_mode: OpenSSL::SSL::VERIFY_NONE,
+      open_timeout: 1,
+      read_timeout: 1
+    ) do |http|
+      request = Net::HTTP::Get.new(path)
+      request["Host"] = host
+      http.request(request)
+    end
+  end
+
+  def peer_certificate(port, hostname)
+    tcp_socket = TCPSocket.new("127.0.0.1", port)
+    ssl_context = OpenSSL::SSL::SSLContext.new
+    ssl_context.verify_mode = OpenSSL::SSL::VERIFY_NONE
+    ssl_socket = OpenSSL::SSL::SSLSocket.new(tcp_socket, ssl_context)
+    ssl_socket.hostname = hostname if ssl_socket.respond_to?(:hostname=)
+    ssl_socket.connect
+    ssl_socket.peer_cert
+  ensure
+    ssl_socket&.close
+    tcp_socket&.close
+  end
+
+  def assert_certificate_domain(certificate, domain)
+    alt_names = certificate.extensions
+      .select { |extension| extension.oid == "subjectAltName" }
+      .flat_map { |extension| extension.value.split(/,\s*/) }
+
+    assert_includes alt_names, "DNS:#{domain}"
+  end
+end

data/gems/server/test/helpers/local_acme.rb

@@ -0,0 +1,218 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "json"
+require "net/http"
+require "openssl"
+require "rbconfig"
+require "shellwords"
+require "timeout"
+require "tmpdir"
+
+class LocalAcmeAuthority
+  PEBBLE_VERSION = "v2.10.1"
+
+  attr_reader :directory_url, :cache_dir, :http_port, :tls_port
+
+  def self.available?
+    system("go version > /dev/null 2>&1")
+  end
+
+  def initialize
+    @workspace = Dir.mktmpdir("itsi-local-acme-")
+    @bin_dir = File.join(@workspace, "bin")
+    @cache_dir = File.join(@workspace, "cache")
+    FileUtils.mkdir_p(@bin_dir)
+    FileUtils.mkdir_p(@cache_dir)
+    @pids = []
+  end
+
+  def start
+    install_binaries
+    resolve_module_paths
+
+    @acme_port = free_port
+    @management_port = free_port
+    @dns_port = free_port
+    @dns_management_port = free_port
+    @http_port = free_port
+    @tls_port = free_port
+
+    write_pebble_config
+    start_challtestsrv
+    start_pebble
+
+    @directory_url = "https://localhost:#{@acme_port}/dir"
+    wait_for_https(@directory_url, @pebble_api_ca_path)
+  end
+
+  def stop
+    @pids.reverse_each do |pid|
+      begin
+        Process.kill("TERM", pid)
+      rescue Errno::ESRCH
+        next
+      end
+
+      begin
+        Timeout.timeout(5) { Process.wait(pid) }
+      rescue Timeout::Error
+        begin
+          Process.kill("KILL", pid)
+        rescue Errno::ESRCH
+          nil
+        end
+        begin
+          Process.wait(pid)
+        rescue Errno::ECHILD
+          nil
+        end
+      rescue Errno::ECHILD
+        nil
+      end
+    end
+    FileUtils.remove_entry(@workspace) if File.exist?(@workspace)
+  end
+
+  def env
+    {
+      "ITSI_ACME_DIRECTORY_URL" => @directory_url,
+      "ITSI_ACME_CA_PEM_PATH" => @pebble_api_ca_path,
+      "ITSI_ACME_CACHE_DIR" => @cache_dir
+    }
+  end
+
+  private
+
+  def install_binaries
+    install_go_binary("github.com/letsencrypt/pebble/v2/cmd/pebble", "pebble")
+    install_go_binary("github.com/letsencrypt/pebble/v2/cmd/pebble-challtestsrv", "pebble-challtestsrv")
+  end
+
+  def install_go_binary(package_name, binary_name)
+    destination = File.join(@bin_dir, binary_name)
+    return if File.exist?(destination)
+
+    system(
+      {
+        "GOBIN" => @bin_dir
+      },
+      "go", "install", "#{package_name}@#{PEBBLE_VERSION}",
+      exception: true
+    )
+  end
+
+  def resolve_module_paths
+    module_json = JSON.parse(`go mod download -json github.com/letsencrypt/pebble/v2@#{PEBBLE_VERSION}`)
+    @pebble_module_dir = module_json.fetch("Dir")
+    @pebble_api_ca_path = File.join(@pebble_module_dir, "test/certs/pebble.minica.pem")
+    @pebble_tls_cert_path = File.join(@pebble_module_dir, "test/certs/localhost/cert.pem")
+    @pebble_tls_key_path = File.join(@pebble_module_dir, "test/certs/localhost/key.pem")
+  end
+
+  def write_pebble_config
+    @pebble_config_path = File.join(@workspace, "pebble-config.json")
+    File.write(
+      @pebble_config_path,
+      JSON.pretty_generate(
+        {
+          pebble: {
+            listenAddress: "0.0.0.0:#{@acme_port}",
+            managementListenAddress: "0.0.0.0:#{@management_port}",
+            certificate: @pebble_tls_cert_path,
+            privateKey: @pebble_tls_key_path,
+            httpPort: @http_port,
+            tlsPort: @tls_port,
+            ocspResponderURL: "",
+            externalAccountBindingRequired: false,
+            domainBlocklist: [],
+            retryAfter: {
+              authz: 1,
+              order: 1
+            },
+            keyAlgorithm: "ecdsa"
+          }
+        }
+      )
+    )
+  end
+
+  def start_challtestsrv
+    spawn_process(
+      [
+        File.join(@bin_dir, "pebble-challtestsrv"),
+        "-dnsserver", "127.0.0.1:#{@dns_port}",
+        "-management", "127.0.0.1:#{@dns_management_port}",
+        "-http01", "",
+        "-https01", "",
+        "-tlsalpn01", "",
+        "-defaultIPv4", "127.0.0.1",
+        "-defaultIPv6", ""
+      ],
+      "challtestsrv"
+    )
+  end
+
+  def start_pebble
+    spawn_process(
+      [
+        File.join(@bin_dir, "pebble"),
+        "-config", @pebble_config_path,
+        "-dnsserver", "127.0.0.1:#{@dns_port}",
+        "-strict=false"
+      ],
+      "pebble",
+      {
+        "PEBBLE_VA_NOSLEEP" => "1",
+        "PEBBLE_WFE_NONCEREJECT" => "0",
+        "PEBBLE_AUTHZREUSE" => "0"
+      }
+    )
+  end
+
+  def spawn_process(command, name, env = {})
+    log_path = File.join(@workspace, "#{name}.log")
+    log = File.open(log_path, "w")
+    pid = Process.spawn(
+      env,
+      *command,
+      out: log,
+      err: log
+    )
+    @pids << pid
+  end
+
+  def free_port
+    server = TCPServer.new("127.0.0.1", 0)
+    port = server.addr[1]
+    server.close
+    port
+  end
+
+  def wait_for_https(url, ca_file)
+    uri = URI(url)
+    store = OpenSSL::X509::Store.new
+    store.add_file(ca_file)
+
+    Timeout.timeout(20) do
+      loop do
+        begin
+          Net::HTTP.start(
+            uri.host,
+            uri.port,
+            use_ssl: true,
+            cert_store: store,
+            verify_mode: OpenSSL::SSL::VERIFY_PEER,
+            open_timeout: 1,
+            read_timeout: 1
+          ) do |http|
+            response = http.get(uri.request_uri)
+            return if response.code.to_i < 500
+          end
+        rescue StandardError
+          sleep 0.1
+        end
+      end
+    end
+  end
+end