http_loader 0.10.3

data/lib/http_loader/client.rb

@@ -0,0 +1,135 @@
+# typed: strong
+# frozen_string_literal: true
+
+require 'sorbet-runtime'
+require 'net/http'
+require 'uri'
+require 'openssl'
+require 'async'
+require 'async/semaphore'
+require 'time'
+require_relative 'client/config'
+require_relative 'client/logger'
+require_relative 'client/target_manager'
+require_relative 'client/slowloris'
+require_relative 'client/http_session'
+require_relative 'client/error_handler'
+
+module HttpLoader
+  # Master client class coordinating connection pools through Async Engine.
+  class Client
+    extend T::Sig
+    include ErrorHandler
+
+    sig { params(config: Config).void }
+    def initialize(config)
+      @config = config
+      @logger = T.let(Logger.new(config.verbose), Logger)
+      @target_manager = T.let(TargetManager.new(config), TargetManager)
+      @slow_sess = T.let(Slowloris.new(config, @logger), Slowloris)
+      @http_sess = T.let(HttpSession.new(config, @logger), HttpSession)
+    end
+
+    sig { void }
+    def start
+      log_startup_message
+      trap('INT') { exit(0) }
+      @logger.setup_files!
+
+      Async { |task| run_engine(task) }
+    ensure
+      @logger.flush_synchronously!
+    end
+
+    private
+
+    sig { params(task: T.untyped).void }
+    def run_engine(task)
+      logger_t = @logger.run_task(task)
+      sem = Async::Semaphore.new(@config.max_concurrent_connections, parent: task)
+
+      conn_tasks = Array.new(@config.connections) do |i|
+        perform_sleep(calc_ramp, task: task) if calc_ramp.positive?
+        sem.async { exec_conn(i) }
+      end
+
+      conn_tasks.each(&:wait)
+      logger_t.stop
+    end
+
+    sig { void }
+    def log_startup_message
+      puts "[Client] Starting #{@config.connections} #{@target_manager.protocol_label} connections to targeted urls..."
+      puts '[Client] Note: Output of individual pings is suppressed.' unless @config.verbose
+    end
+
+    sig { returns(Float) }
+    def calc_ramp
+      if @config.ramp_up.positive?
+        @config.ramp_up.to_f / @config.connections
+      elsif @config.connections_per_second.positive?
+        1.0 / @config.connections_per_second
+      else
+        0.0
+      end
+    end
+
+    sig { params(dur: Float, task: T.untyped).void }
+    def perform_sleep(dur, task: nil)
+      task ? task.sleep(dur) : sleep(dur)
+    end
+
+    sig { params(base: Float).returns(Float) }
+    def calc_sleep(base)
+      return base if @config.jitter.zero?
+
+      v = base * @config.jitter
+      [0.0, base + rand(-v..v)].max
+    end
+
+    sig { params(idx: Integer).void }
+    def exec_conn(idx)
+      loop do
+        run_session(idx, Time.now)
+        break unless @config.reopen_closed_connections
+
+        sleep(calc_sleep(@config.reopen_interval))
+      end
+    end
+
+    sig { params(idx: Integer, start_t: Time).void }
+    def run_session(idx, start_t)
+      ctx = @target_manager.context_for(idx)
+      uri = T.cast(ctx[:uri], URI::Generic)
+
+      start_http(uri, fetch_opts(idx, ctx)) do |http|
+        http.max_retries = 0 if http.respond_to?(:max_retries=)
+        @logger.info("[Client #{idx}] Connection established to #{uri.host}.")
+        dispatch_sess(idx, uri, http, start_t)
+      end
+      @logger.info("[Client #{idx}] Connection gracefully closed.")
+    rescue StandardError => e
+      handle_err(idx, e)
+    end
+
+    sig { params(idx: Integer, ctx: T::Hash[Symbol, T.untyped]).returns(T::Hash[Symbol, T.untyped]) }
+    def fetch_opts(idx, ctx)
+      args = T.cast(ctx[:http_args], T::Hash[Symbol, T.untyped])
+      @target_manager.http_opts_for(idx, args)
+    end
+
+    sig { params(uri: URI::Generic, opts: T::Hash[Symbol, T.untyped], block: T.proc.params(arg: Net::HTTP).void).void }
+    def start_http(uri, opts, &block)
+      Net::HTTP.start(T.must(uri.host), uri.port, **opts, &block)
+    end
+
+    sig { params(idx: Integer, uri: URI::Generic, http: Net::HTTP, start_t: Time).void }
+    def dispatch_sess(idx, uri, http, start_t)
+      if @config.slowloris_delay.positive?
+        @slow_sess.run(idx, uri, http, start_t)
+      else
+        @http_sess.run(idx, uri, http, start_t)
+      end
+    end
+  end
+end

data/lib/http_loader/harness/config.rb

@@ -0,0 +1,18 @@
+# typed: strong
+# frozen_string_literal: true
+
+require 'sorbet-runtime'
+
+module HttpLoader
+  class Harness
+    # Config parameters for Harness manager execution natively.
+    class Config < T::Struct
+      const :connections, Integer
+      const :target_urls, T::Array[String], default: []
+      const :use_https, T::Boolean, default: false
+      const :client_args, T::Array[String], default: []
+      const :export_json, T.nilable(String), default: nil
+      const :target_duration, Float, default: 0.0
+    end
+  end
+end

data/lib/http_loader/harness/formatter.rb

@@ -0,0 +1,82 @@
+# typed: strong
+# frozen_string_literal: true
+
+require 'sorbet-runtime'
+
+module HttpLoader
+  class Harness
+    # Formatter handles printing load test statistics dynamically and robustly.
+    module Formatter
+      extend T::Sig
+      extend T::Helpers
+
+      requires_ancestor { HttpLoader::Harness }
+
+      sig { void }
+      def print_startup_banner
+        msg = if @config.target_urls.size > 1
+                "**MULTIPLE TARGETS** (#{@config.target_urls.size} URLs)."
+              elsif @config.target_urls.size == 1
+                "**EXTERNAL URL** #{@config.target_urls.first}."
+              elsif @config.use_https
+                '**HTTPS**.'
+              else
+                '**HTTP**.'
+              end
+        puts "[Harness] Starting test with #{@config.connections} connections to #{msg}"
+      end
+
+      sig { void }
+      def print_table_header
+        puts '[Harness] Monitoring resources (Press Ctrl+C to stop)...'
+        puts '-' * 125
+        puts 'Time (UTC) | Real Conns  | Srv CPU/Thrds    | Srv Mem        | Srv Mem/Conn   | ' \
+             'Cli CPU/Thrds    | Cli Mem        | Cli Mem/Conn  '
+        puts '-' * 125
+      end
+
+      sig { params(params: T::Hash[Symbol, T.untyped]).void }
+      def log_table_row(params)
+        puts format('%<t>-10s | %<ac>-11s | %<sc>-16s | %<sm>-14s | %<sk>-14s | %<cc>-16s | %<cm>-14s | %<ck>-14s',
+                    t: params[:t], ac: params[:ac], sc: params[:sc], sm: params[:sm],
+                    sk: params[:sk], cc: params[:cc], cm: params[:cm], ck: params[:ck])
+      end
+
+      sig { params(kilo: Float, connections: Integer, pid: T.nilable(Integer)).returns(String) }
+      def format_kb_conn(kilo, connections, pid)
+        return 'EXTERNAL' if pid.nil?
+        return 'N/A' if connections.zero? || connections.negative?
+
+        "#{(kilo / connections).round(2)} KB"
+      end
+
+      sig { params(active: Integer, c_cpu: String, c_th: Integer, c_m: String).void }
+      def print_combined_stats(active, c_cpu, c_th, c_m)
+        s_cpu, s_mem, s_th, s_conn = extract_server_stats
+        _cc, _cm, c_kb, _ct = @monitor.process_stats(@pm.client_pid)
+        c_conn = format_kb_conn(c_kb, active, @pm.client_pid)
+
+        log_table_row(
+          t: Time.now.utc.strftime('%H:%M:%S'), ac: active.to_s,
+          sc: "#{s_cpu}% / #{s_th}", sm: s_mem, sk: s_conn,
+          cc: "#{c_cpu}% / #{c_th}", cm: c_m, ck: c_conn
+        )
+      end
+
+      sig { returns([String, String, Integer, String]) }
+      def extract_server_stats
+        s_cpu, s_mem, s_kb, s_th = @monitor.process_stats(@pm.server_pid)
+        active_s = @monitor.count_established_connections(@pm.server_pid)
+        s_conn = format_kb_conn(s_kb, active_s, @pm.server_pid)
+        [s_cpu, s_mem, s_th, s_conn]
+      end
+
+      sig { returns([Integer, String, Integer, String]) }
+      def extract_client_stats
+        c_cpu, c_mem, _c_kb, c_th = @monitor.process_stats(@pm.client_pid)
+        active_c = @monitor.count_established_connections(@pm.client_pid)
+        [active_c, c_cpu, c_th, c_mem]
+      end
+    end
+  end
+end

data/lib/http_loader/harness/process_manager.rb

@@ -0,0 +1,81 @@
+# typed: strong
+# frozen_string_literal: true
+
+require 'sorbet-runtime'
+
+module HttpLoader
+  class Harness
+    # Manages child process lifecycles natively and checks status dynamically.
+    class ProcessManager
+      extend T::Sig
+
+      sig { returns(T.nilable(Integer)) }
+      attr_reader :server_pid, :client_pid
+
+      sig { params(config: HttpLoader::Harness::Config).void }
+      def initialize(config)
+        @config = config
+        @server_pid = T.let(nil, T.nilable(Integer))
+        @client_pid = T.let(nil, T.nilable(Integer))
+        @log_dir = T.let(File.expand_path('../../../logs', __dir__), String)
+      end
+
+      sig { void }
+      def spawn_processes
+        FileUtils.mkdir_p(@log_dir)
+        spawn_server unless @config.target_urls.any?
+        spawn_client
+      end
+
+      sig { void }
+      def cleanup
+        begin
+          Process.kill('INT', T.must(@server_pid)) if @server_pid
+        rescue StandardError; nil
+        end
+        begin
+          Process.kill('INT', T.must(@client_pid)) if @client_pid
+        rescue StandardError; nil
+        end
+      end
+
+      sig { returns(T::Boolean) }
+      def missing_process?
+        return true if @client_pid && dead?(@client_pid)
+        return true if @server_pid && dead?(@server_pid)
+
+        false
+      end
+
+      private
+
+      sig { void }
+      def spawn_server
+        server_cmd = ['ruby', 'bin/http_loader', 'server']
+        server_cmd << '--https' if @config.use_https
+        @server_pid = Process.spawn(*server_cmd, out: File.join(@log_dir, 'server.log'),
+                                                 err: File.join(@log_dir, 'server.err'))
+        puts "[Harness] Started server with PID #{@server_pid}"
+        sleep(2)
+      end
+
+      sig { void }
+      def spawn_client
+        client_cmd = ['ruby', 'bin/http_loader', 'client']
+        client_cmd += @config.client_args.empty? ? ["--connections_count=#{@config.connections}"] : @config.client_args
+
+        @client_pid = Process.spawn(*client_cmd, out: File.join(@log_dir, 'client.log'),
+                                                 err: File.join(@log_dir, 'client.err'))
+        puts "[Harness] Started client with PID #{@client_pid}"
+      end
+
+      sig { params(pid: Integer).returns(T::Boolean) }
+      def dead?(pid)
+        Process.getpgid(pid)
+        false
+      rescue Errno::ESRCH
+        true
+      end
+    end
+  end
+end

data/lib/http_loader/harness/resource_monitor.rb

@@ -0,0 +1,132 @@
+# typed: strong
+# frozen_string_literal: true
+
+require 'sorbet-runtime'
+require 'open3'
+
+module HttpLoader
+  class Harness
+    # Monitors CPU, memory, and established connections for running processes.
+    class ResourceMonitor
+      extend T::Sig
+
+      sig { void }
+      def initialize
+        @cpu_prev = T.let({}, T::Hash[Integer, T::Hash[Symbol, T.untyped]])
+      end
+
+      sig { params(pid: T.nilable(Integer)).returns([String, String, Float, Integer]) }
+      def process_stats(pid)
+        return ['EXTERNAL', 'EXTERNAL', 0.0, 0] if pid.nil?
+
+        if File.exist?("/proc/#{pid}/stat") && File.exist?("/proc/#{pid}/statm")
+          native_linux_stats(pid)
+        else
+          fallback_ps_stats(pid)
+        end
+      rescue StandardError
+        fallback_ps_stats(pid)
+      end
+
+      sig { params(pid: T.nilable(Integer)).returns(Integer) }
+      def count_established_connections(pid)
+        return 0 if pid.nil?
+
+        if File.directory?("/proc/#{pid}/fd")
+          count_linux_connections(pid)
+        else
+          count_lsof_connections(pid)
+        end
+      rescue StandardError
+        0
+      end
+
+      private
+
+      sig { params(pid: Integer).returns([String, String, Float, Integer]) }
+      def native_linux_stats(pid)
+        rss_kb = read_rss_kb(pid)
+        cpu_perc = read_cpu_perc(pid)
+        [cpu_perc.to_s, "#{(rss_kb / 1024.0).round(2)} MB", rss_kb, read_threads(pid)]
+      end
+
+      sig { params(pid: Integer).returns(Float) }
+      def read_rss_kb(pid)
+        rss_pages = T.must(File.read("/proc/#{pid}/statm").split[1]).to_i
+        (rss_pages * fetch_page_size) / 1024.0
+      end
+
+      sig { params(pid: Integer).returns(Float) }
+      def read_cpu_perc(pid)
+        stat_array = File.read("/proc/#{pid}/stat").split
+        calculate_cpu(pid, T.must(stat_array[13]).to_f + T.must(stat_array[14]).to_f)
+      end
+
+      sig { params(pid: Integer).returns(Integer) }
+      def read_threads(pid)
+        File.read("/proc/#{pid}/status").match(/Threads:\s+(\d+)/)&.[](1)&.to_i || 1
+      end
+
+      sig { returns(Integer) }
+      def fetch_page_size
+        return 4096 unless File.exist?('/usr/bin/getconf')
+
+        out, _s = Open3.capture2('getconf PAGE_SIZE')
+        out.to_i.positive? ? out.to_i : 4096
+      rescue StandardError
+        4096
+      end
+
+      sig { params(pid: Integer, total_ticks: Float).returns(Float) }
+      def calculate_cpu(pid, total_ticks)
+        prev = @cpu_prev[pid]
+        now = Time.now.utc
+        cpu_perc = 0.0
+
+        if prev && (time_diff = now - prev[:time]).positive?
+          cpu_perc = (((total_ticks - prev[:ticks]) / 100.0) / time_diff * 100).round(1)
+        end
+
+        @cpu_prev[pid] = { ticks: total_ticks, time: now }
+        cpu_perc
+      end
+
+      sig { params(pid: Integer).returns([String, String, Float, Integer]) }
+      def fallback_ps_stats(pid)
+        out, _s = Open3.capture2('ps', '-o', '%cpu,rss', '-p', pid.to_s)
+        lines = out.strip.split("\n")
+        return ['N/A', 'N/A', 0.0, 0] if lines.size < 2
+
+        cpu, rss_kb_str = T.must(lines[1]).strip.split(/\s+/)
+        rss_kb_val = T.must(rss_kb_str).to_f
+
+        [T.must(cpu), "#{(rss_kb_val / 1024.0).round(2)} MB", rss_kb_val, fallback_threads_count(pid)]
+      rescue StandardError
+        ['N/A', 'N/A', 0.0, 0]
+      end
+
+      sig { params(pid: Integer).returns(Integer) }
+      def fallback_threads_count(pid)
+        out, _s = Open3.capture2("ps -M -p #{pid}")
+        [out.strip.split("\n").size - 1, 0].max
+      rescue StandardError
+        1
+      end
+
+      sig { params(pid: Integer).returns(Integer) }
+      def count_linux_connections(pid)
+        Dir.glob("/proc/#{pid}/fd/*").count do |fd_path|
+          File.readlink(fd_path).start_with?('socket:[')
+        rescue StandardError
+          false
+        end
+      end
+
+      sig { params(pid: Integer).returns(Integer) }
+      def count_lsof_connections(pid)
+        out, _s = Open3.capture2("lsof -p #{pid} -n -P")
+        out.scan('ESTABLISHED').count
+      end
+    end
+  end
+end

data/lib/http_loader/harness/telemetry.rb

@@ -0,0 +1,76 @@
+# typed: strong
+# frozen_string_literal: true
+
+require 'sorbet-runtime'
+require 'json'
+
+module HttpLoader
+  class Harness
+    # Exports telemetry and logs bottlenecks asynchronously and synchronously at termination.
+    class Telemetry
+      extend T::Sig
+
+      sig { params(log_dir: String, export_json: T.nilable(String)).void }
+      def initialize(log_dir, export_json)
+        @log_dir = log_dir
+        @export_json = export_json
+      end
+
+      sig { params(peak_connections: Integer, start_time: Time).void }
+      def export!(peak_connections, start_time)
+        return unless @export_json
+
+        payload = generate_payload(peak_connections, start_time, count_bottlenecks(read_logs))
+        File.write(@export_json, JSON.generate(payload))
+        puts "[Harness] Telemetry JSON securely sinked to #{@export_json}."
+      end
+
+      sig { void }
+      def check_bottlenecks!
+        errors = build_bottleneck_messages(count_bottlenecks(read_logs))
+        puts format('   => BOTTLENECK ACTIVE: %s', errors.join(' | ')) if errors.any?
+      rescue StandardError
+        nil
+      end
+
+      private
+
+      sig { params(peak: Integer, start: Time, counts: T::Hash[Symbol, Integer]).returns(T::Hash[Symbol, T.untyped]) }
+      def generate_payload(peak, start, counts)
+        {
+          peak_connections: peak, test_duration_seconds: (Time.now.utc - start).round(2),
+          errors: { emfile: counts[:emfile], eaddrnotavail: counts[:eaddr_count], thread_limit: counts[:thread_errors] }
+        }
+      end
+
+      sig { returns(String) }
+      def read_logs
+        begin
+          File.read(File.join(@log_dir, 'client.log'))
+        rescue StandardError; ''
+        end + begin
+          File.read(File.join(@log_dir, 'client.err'))
+        rescue StandardError; ''
+        end
+      end
+
+      sig { params(log_text: String).returns(T::Hash[Symbol, Integer]) }
+      def count_bottlenecks(log_text)
+        {
+          emfile: log_text.scan('ERROR_EMFILE').size,
+          eaddr_count: log_text.scan('ERROR_EADDRNOTAVAIL').size,
+          thread_errors: log_text.scan('ERROR_THREADLIMIT').size
+        }
+      end
+
+      sig { params(counts: T::Hash[Symbol, Integer]).returns(T::Array[String]) }
+      def build_bottleneck_messages(counts)
+        errors = []
+        errors << "[OS FDs Limit: #{counts[:emfile]} EMFILE]" if counts[:emfile].positive?
+        errors << "[OS Ports Limit: #{counts[:eaddr_count]} EADDRNOTAVAIL]" if counts[:eaddr_count].positive?
+        errors << "[OS Thread Limit: #{counts[:thread_errors]} ThreadError]" if counts[:thread_errors].positive?
+        errors
+      end
+    end
+  end
+end

data/lib/http_loader/harness.rb

@@ -0,0 +1,110 @@
+# typed: strong
+# frozen_string_literal: true
+
+require 'sorbet-runtime'
+require 'fileutils'
+require_relative 'harness/config'
+require_relative 'harness/resource_monitor'
+require_relative 'harness/telemetry'
+require_relative 'harness/process_manager'
+require_relative 'harness/formatter'
+
+module HttpLoader
+  # Harness orchestrates the entire load testing lifecycle seamlessly and securely.
+  class Harness
+    extend T::Sig
+    include Formatter
+
+    sig { params(config: Config).void }
+    def initialize(config)
+      @config = config
+      @start_time = T.let(Time.now.utc, Time)
+      @peak_connections = T.let(0, Integer)
+      log_dir = T.let(File.expand_path('../../logs', __dir__), String)
+      @telemetry = T.let(Telemetry.new(log_dir, config.export_json), Telemetry)
+      @pm = T.let(ProcessManager.new(config), ProcessManager)
+      @monitor = T.let(ResourceMonitor.new, ResourceMonitor)
+    end
+
+    sig { void }
+    def start
+      $stdout.sync = true
+      print_startup_banner
+      bump_file_limits
+
+      run_lifecycle
+    end
+
+    private
+
+    sig { void }
+    def run_lifecycle
+      @pm.spawn_processes
+      trap('INT') do
+        puts "\n[Harness] Caught interrupt, cleaning up processes..."
+        @pm.cleanup
+        exit(0)
+      end
+      monitor_resources
+    ensure
+      @telemetry.export!(@peak_connections, @start_time)
+      @pm.cleanup
+    end
+
+    sig { void }
+    def monitor_resources
+      print_table_header
+      @start_time = Time.now.utc
+
+      loop do
+        break if duration_exceeded?
+
+        tick_result = tick_failed?
+        break if tick_result
+
+        @telemetry.check_bottlenecks!
+        sleep(2)
+      end
+    end
+
+    sig { returns(T::Boolean) }
+    def duration_exceeded?
+      elapsed = Time.now.utc - @start_time
+      return false unless @config.target_duration.positive? && elapsed >= @config.target_duration
+
+      puts "[Harness] Target duration mathematically reached (#{@config.target_duration}s). auto-shutdown."
+      true
+    end
+
+    sig { returns(T::Boolean) }
+    def tick_failed?
+      active_c, c_cpu, c_th, c_m = extract_client_stats
+      @peak_connections = [@peak_connections, active_c].max
+
+      return true if missing_socket?(active_c, c_cpu, c_th, c_m)
+      return true if @pm.missing_process?
+
+      print_combined_stats(active_c, c_cpu, c_th, c_m)
+      false
+    end
+
+    sig { void }
+    def bump_file_limits
+      Process.setrlimit(Process::RLIMIT_NOFILE, @config.connections + 1024)
+    rescue Errno::EPERM
+      puts '[Harness] Warning: Could not set RLIMIT_NOFILE automatically.'
+    end
+
+    sig { params(active: Integer, c_cpu: String, c_th: Integer, c_m: String).returns(T::Boolean) }
+    def missing_socket?(active, c_cpu, c_th, c_m)
+      return false unless @config.target_urls.any? && @peak_connections.positive? && active.zero?
+
+      log_table_row(
+        t: Time.now.utc.strftime('%H:%M:%S'), ac: active, sc: 'EXTERNAL', sm: 'N/A',
+        sk: 'N/A', cc: "#{c_cpu}% / #{c_th}T", cm: c_m, ck: 'N/A'
+      )
+      puts "\n[Harness] \u26A0\uFE0F EXTERNAL SERVER DISCONNECTED!"
+      true
+    end
+  end
+end