jmeter_perf 1.0.9

data/lib/jmeter_perf/report/summary.rb

@@ -0,0 +1,268 @@
+require "csv"
+require "timeout"
+require_relative "../helpers/running_statistics"
+module JmeterPerf
+  module Report
+    # Summary provides a statistical analysis of performance test results by processing
+    # JMeter JTL files. It calculates metrics such as average response time, error percentage,
+    # min/max response times, and percentiles, helping to understand the distribution of
+    # response times across requests.
+    # @note This class uses a TDigest data structure to keep statistics "close enough". Accuracy is not guaranteed.
+    #
+
+    # @!attribute [rw] avg
+    #   @return [Float] the average response time
+    # @!attribute [rw] error_percentage
+    #   @return [Float] the error percentage across all requests
+    # @!attribute [rw] max
+    #   @return [Integer] the maximum response time encountered
+    # @!attribute [rw] min
+    #   @return [Integer] the minimum response time encountered
+    # @!attribute [rw] p10
+    #   @return [Float] the 10th percentile of response times
+    # @!attribute [rw] p50
+    #   @return [Float] the median (50th percentile) of response times
+    # @!attribute [rw] p95
+    #   @return [Float] the 95th percentile of response times
+    # @!attribute [rw] requests_per_minute
+    #   @return [Float] the requests per minute rate
+    # @!attribute [rw] response_codes
+    #   @return [Hash<String, Integer>] a hash of response codes with their respective counts
+    # @!attribute [rw] standard_deviation
+    #   @return [Float] the standard deviation of response times
+    # @!attribute [rw] total_bytes
+    #   @return [Integer] the total number of bytes received
+    # @!attribute [rw] total_elapsed_time
+    #   @return [Integer] the total elapsed time in milliseconds
+    # @!attribute [rw] total_errors
+    #   @return [Integer] the total number of errors encountered
+    # @!attribute [rw] total_latency
+    #   @return [Integer] the total latency time across all requests
+    # @!attribute [rw] total_requests
+    #   @return [Integer] the total number of requests processed
+    # @!attribute [rw] total_sent_bytes
+    #   @return [Integer] the total number of bytes sent
+    # @!attribute [rw] csv_error_lines
+    #   @return [Array<Integer>] the line numbers of where CSV errors were encountered
+    # @!attribute [rw] total_run_time
+    #   @return [Integer] the total run time in seconds
+    # @!attribute [rw] name
+    #   @return [String] the name of the summary, derived from the file path if not provided
+    # @!attribute [rw] response_codes
+    #   @return [Hash<String, Integer>] a hash of response codes with their respective counts
+    # @!attribute [rw] csv_error_lines
+    #   @return [Array<Integer>] the line numbers of where CSV errors were encountered
+    class Summary
+      # JTL file headers used for parsing CSV rows
+      JTL_HEADER = %i[
+        timeStamp
+        elapsed
+        label
+        responseCode
+        responseMessage
+        threadName
+        dataType
+        success
+        failureMessage
+        bytes
+        sentBytes
+        grpThreads
+        allThreads
+        URL
+        Latency
+        IdleTime
+        Connect
+      ]
+
+      # @return [Hash<String, Symbol>] a mapping of CSV headers to their corresponding attribute symbols.
+      CSV_HEADER_MAPPINGS = {
+        "Name" => :name,
+        "Average Response Time" => :avg,
+        "Error Percentage" => :error_percentage,
+        "Max Response Time" => :max,
+        "Min Response Time" => :min,
+        "10th Percentile" => :p10,
+        "Median (50th Percentile)" => :p50,
+        "95th Percentile" => :p95,
+        "Requests Per Minute" => :requests_per_minute,
+        "Standard Deviation" => :standard_deviation,
+        "Total Run Time" => :total_run_time,
+        "Total Bytes" => :total_bytes,
+        "Total Elapsed Time" => :total_elapsed_time,
+        "Total Errors" => :total_errors,
+        "Total Latency" => :total_latency,
+        "Total Requests" => :total_requests,
+        "Total Sent Bytes" => :total_sent_bytes
+        # "Response Code 200" => :response_codes["200"],
+        # "Response Code 500" => :response_codes["500"] etc.
+      }
+
+      attr_accessor(*CSV_HEADER_MAPPINGS.values)
+      # Response codes have multiple keys, so we need to handle them separately
+      attr_accessor :response_codes
+      # CSV Error Lines are an array of integers that get delimited by ":" when written to the CSV
+      attr_accessor :csv_error_lines
+      alias_method :rpm, :requests_per_minute
+      alias_method :std, :standard_deviation
+      alias_method :median, :p50
+
+      # Reads a generated CSV report and sets all appropriate attributes.
+      #
+      # @param csv_path [String] the file path of the CSV report to read
+      # @return [Summary] a new Summary instance with the parsed data
+      def self.read(csv_path)
+        summary = new(file_path: csv_path)
+        CSV.foreach(csv_path, headers: true) do |row|
+          metric = row["Metric"]
+          value = row["Value"]
+
+          if metric == "Name"
+            summary.name = value
+          elsif metric.start_with?("Response Code")
+            code = metric.split.last
+            summary.response_codes[code] = value.to_i
+          elsif metric == "CSV Errors"
+            summary.csv_error_lines = value.split(":").map(&:to_i)
+          elsif CSV_HEADER_MAPPINGS.key?(metric)
+            summary.public_send(:"#{CSV_HEADER_MAPPINGS[metric]}=", value.include?(".") ? value.to_f : value.to_i)
+          end
+        end
+        summary
+      end
+
+      # Initializes a new Summary instance for analyzing performance data.
+      #
+      # @param file_path [String] the file path of the performance file to summarize. Either a JTL or CSV file.
+      # @param name [String, nil] an optional name for the summary, derived from the file path if not provided (default: nil)
+      # @param jtl_read_timeout [Integer] the maximum number of seconds to wait for a line read (default: 3)
+      def initialize(file_path:, name: nil, jtl_read_timeout: 3)
+        @name = name || file_path.to_s.tr("/", "_")
+        @jtl_read_timeout = jtl_read_timeout
+        @finished = false
+        @running_statistics_helper = JmeterPerf::Helpers::RunningStatistisc.new
+
+        @max = 0
+        @min = 1_000_000
+        @response_codes = Hash.new { |h, k| h[k.to_s] = 0 }
+        @total_bytes = 0
+        @total_elapsed_time = 0
+        @total_errors = 0
+        @total_latency = 0
+        @total_requests = 0
+        @total_sent_bytes = 0
+        @csv_error_lines = []
+
+        @file_path = file_path
+
+        @start_time = nil
+        @end_time = nil
+      end
+
+      # Marks the summary as finished, allowing any pending asynchronous operations to complete.
+      #
+      # @return [void]
+      def finish!
+        @finished = true
+        @processing_jtl_thread.join if @processing_jtl_thread&.alive?
+      end
+
+      # Generates a CSV report with the given output file.
+      #
+      # The CSV report includes the following:
+      # - A header row with "Metric" and "Value".
+      # - Rows for each metric and its corresponding value from `CSV_HEADER_MAPPINGS`.
+      # - Rows for each response code and its count from `@response_codes`.
+      # - A row for CSV errors, concatenated with ":".
+      #
+      # @param output_file [String] The path to the output CSV file.
+      # @return [void]
+      def write_csv(output_file)
+        CSV.open(output_file, "wb") do |csv|
+          csv << ["Metric", "Value"]
+          CSV_HEADER_MAPPINGS.each do |metric, value|
+            csv << [metric, public_send(value)]
+          end
+          @response_codes.each do |code, count|
+            csv << ["Response Code #{code}", count]
+          end
+
+          csv << ["CSV Errors", @csv_error_lines.join(":")]
+        end
+      end
+
+      # Starts streaming and processing JTL file content asynchronously.
+      #
+      # @return [Thread] a thread that handles the asynchronous file streaming and parsing
+      def stream_jtl_async
+        @processing_jtl_thread = Thread.new do
+          sleep 0.1 until File.exist?(@file_path) # Wait for the file to be created
+
+          File.open(@file_path, "r") do |file|
+            file.seek(0, IO::SEEK_END)
+            until @finished && file.eof?
+              line = file.gets
+              next unless line # Skip if no line was read
+
+              # Process only if the line is complete (ends with a newline)
+              read_until_complete_line(file, line)
+            end
+          end
+        end
+      end
+
+      # Summarizes the collected data by calculating statistical metrics and error rates.
+      #
+      # @return [void]
+      def summarize_data!
+        @p10, @p50, @p95 = @running_statistics_helper.get_percentiles(0.1, 0.5, 0.95)
+        @error_percentage = (@total_errors.to_f / @total_requests) * 100
+        @avg = @running_statistics_helper.avg
+        @total_run_time = ((@end_time - @start_time) / 1000).to_f  # Convert milliseconds to seconds
+        @requests_per_minute = @total_run_time.zero? ? 0 : (@total_requests / @total_run_time) * 60.0
+        @standard_deviation = @running_statistics_helper.std
+      end
+
+      private
+
+      def read_until_complete_line(file, line)
+        return if file.lineno == 1 # Skip the header row
+        Timeout.timeout(@jtl_read_timeout) do
+          until line.end_with?("\n")
+            sleep 0.1
+            line += file.gets.to_s
+          end
+        end
+        parse_csv_row(line)
+      rescue Timeout::Error
+        puts "Timeout waiting for line to complete: #{line}"
+        raise
+      rescue CSV::MalformedCSVError
+        @csv_error_lines << file.lineno
+      end
+
+      def parse_csv_row(line)
+        CSV.parse(line, headers: JTL_HEADER, liberal_parsing: true).each do |row|
+          line_item = row.to_hash
+          elapsed = line_item.fetch(:elapsed).to_i
+          timestamp = line_item.fetch(:timeStamp).to_i
+
+          # Update start and end times
+          @start_time = timestamp if @start_time.nil? || timestamp < @start_time
+          @end_time = timestamp + elapsed if @end_time.nil? || (timestamp + elapsed) > @end_time
+
+          # Continue with processing the row as before...
+          @running_statistics_helper.add_number(elapsed)
+          @total_requests += 1
+          @total_elapsed_time += elapsed
+          @response_codes[line_item.fetch(:responseCode)] += 1
+          @total_errors += (line_item.fetch(:success) == "true") ? 0 : 1
+          @total_bytes += line_item.fetch(:bytes, 0).to_i
+          @total_sent_bytes += line_item.fetch(:sentBytes, 0).to_i
+          @total_latency += line_item.fetch(:Latency).to_i
+          @min = [@min, elapsed].min
+          @max = [@max, elapsed].max
+        end
+      end
+    end
+  end
+end

data/lib/jmeter_perf/tasks/dsl.rake

@@ -0,0 +1,19 @@
+namespace :jmeter_perf do
+  desc "Generate JmeterPerf::DSL methods"
+  task :dsl_generate, [:dsl_dir, :idl_xml_path] do |t, args|
+    require_relative "../helpers/dsl_generator"
+    puts "Generating DSL methods..."
+
+    dsl_dir = Pathname(args[:dsl_dir]).expand_path
+    idl_xml_path = Pathname(args[:idl_xml_path]).expand_path
+
+    generator = JmeterPerf::Helpers::DSLGenerator.new(
+      dsl_dir: dsl_dir,
+      idl_xml_path: idl_xml_path
+    )
+
+    generator.generate
+
+    puts "Finished generating DSL methods."
+  end
+end

data/lib/jmeter_perf/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module JmeterPerf
+  VERSION = "1.0.9"
+end

data/lib/jmeter_perf/views/report_template.html.erb

@@ -0,0 +1,114 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>Comparison Report</title>
+  <style>
+    body {
+      font-family: Arial, sans-serif;
+    }
+    .summary-box {
+      margin-bottom: 20px;
+      padding: 10px;
+      border: 1px solid #ccc;
+      border-radius: 5px;
+      background-color: #f9f9f9;
+    }
+    table {
+      width: 100%;
+      border-collapse: collapse;
+      table-layout: fixed;
+    }
+    td {
+      border: 1px solid black;
+      padding: 8px;
+      text-align: center;
+      word-wrap: break-word;
+    }
+    .header-row td {
+      background-color: #f2f2f2;
+      font-weight: bold;
+    }
+    .bad {
+      background-color: #ffcccc;
+    }
+    .tooltip {
+      position: relative;
+    }
+    .tooltip .tooltiptext {
+      visibility: hidden;
+      width: 150px;
+      background-color: black;
+      color: #fff;
+      text-align: center;
+      border-radius: 6px;
+      padding: 5px;
+      position: absolute;
+      z-index: 1;
+      bottom: 100%;
+      left: 50%;
+      margin-left: -75px;
+      opacity: 0;
+      transition: opacity 0.3s;
+    }
+    .tooltip:hover .tooltiptext {
+      visibility: visible;
+      opacity: 1;
+    }
+  </style>
+</head>
+<body>
+  <% if @comparator.name %>
+    <h2>Comparison Report for <span style="text-decoration: underline;"><%= @comparator.name %></span></h2>
+  <% end %>
+  <div class="summary-box" id="summary-box">
+    <b>Report Generated:</b> <%= Time.now.strftime("%Y-%m-%d %H:%M:%S") %><br>
+    <b>Cohen's D:</b> <%= @comparator.cohens_d %><br>
+    <b>T Statistics:</b> <%= @comparator.t_statistic %><br>
+    <b>Summary:</b> <%= @comparator.human_rating %> in performance
+  </div>
+  <table>
+    <tr class="header-row">
+      <% ['Label', 'Total Requests', 'Total Elapsed Time', 'RPM', 'Errors', 'Error %', 'Min', 'Max', 'Avg', 'SD', 'P10', 'P50', 'P95'].each do |header| %>
+        <td class="tooltip"><%= header %>
+          <span class="tooltiptext"><%= "Description for #{header}" %></span>
+        </td>
+      <% end %>
+    </tr>
+    <% @reports.each_with_index do |report, index| %>
+      <tr>
+        <td><%= index == 0 ? "Base Metric" : "Test Metric" %></td>
+        <td><%= report.total_requests %></td>
+        <td><%= report.total_elapsed_time %></td>
+        <td><%= sprintf('%.2f', report.rpm) %></td>
+        <td class="<%= 'bad' if report.total_errors > 0 %>"><%= report.total_errors %></td>
+        <td class="<%= 'bad' if report.error_percentage > 0.0 %>"><%= sprintf('%.2f', report.error_percentage) %></td>
+        <td><%= report.min %></td>
+        <td><%= report.max %></td>
+        <td><%= sprintf('%.2f', report.avg) %></td>
+        <td><%= sprintf('%.2f', report.std) %></td>
+        <td><%= report.p10 %></td>
+        <td><%= report.p50 %></td>
+        <td><%= report.p95 %></td>
+      </tr>
+    <% end %>
+  </table>
+  <script type="module">
+    import ColorScale from "https://cdn.skypack.dev/color-scales";
+    document.addEventListener('DOMContentLoaded', function () {
+      const summaryBox = document.getElementById('summary-box');
+      const cohenValue = <%= @comparator.cohens_d %>;
+      let color;
+
+      if (cohenValue === 0) {
+        color = 'lightgray'; // Neutral color for zero
+      } else {
+        const colorScale = new ColorScale(-3, 2, ['#FF0000', '#FFFF00', '#008000']);
+        color = colorScale.getColor(cohenValue).toRGBString()
+      }
+
+      summaryBox.style.backgroundColor = color;
+    });
+
+  </script>
+</body>
+</html>

data/lib/jmeter_perf.rb

@@ -0,0 +1,183 @@
+# frozen_string_literal: true
+
+require_relative "jmeter_perf/version"
+lib = File.dirname(File.absolute_path(__FILE__))
+
+Dir.glob(File.join(lib, "jmeter_perf/report/*.rb")).each do |file|
+  require_relative file
+end
+
+Dir.glob(File.join(lib, "jmeter_perf/helpers/*.rb")).each do |file|
+  require_relative file
+end
+
+Dir.glob(File.join(lib, "jmeter_perf/dsl/*.rb")).each do |file|
+  require_relative file
+end
+
+Dir.glob(File.join(lib, "jmeter_perf/extend/**/*.rb")).each do |file|
+  require_relative file
+end
+
+Dir.glob(File.join(lib, "jmeter_perf/plugins/*.rb")).each do |file|
+  require_relative file
+end
+
+# JmeterPerf module for handling performance testing with JMeter.
+# This module provides methods to define and execute JMeter test plans.
+module JmeterPerf
+  # Evaluates the test plan with the given parameters and block.
+  # @see https://github.com/jlurena/jmeter_perf/wiki/1.-DSL-Documentation DSL Documentation
+  # @param params [Hash] Parameters for the test plan (default: `{}`).
+  # @yield The block to define the test plan steps.
+  # @return [JmeterPerf::ExtendedDSL] The DSL instance with the configured test plan.
+  def self.test(params = {}, &block)
+    dsl = JmeterPerf::ExtendedDSL.new(params)
+
+    block_context = eval("self", block.binding, __FILE__, __LINE__)
+    proxy_context = JmeterPerf::Helpers::FallbackContextProxy.new(dsl, block_context)
+    begin
+      block_context.instance_variables.each do |ivar|
+        proxy_context.instance_variable_set(ivar, block_context.instance_variable_get(ivar))
+      end
+      proxy_context.instance_eval(&block)
+    ensure
+      block_context.instance_variables.each do |ivar|
+        block_context.instance_variable_set(ivar, proxy_context.instance_variable_get(ivar))
+      end
+    end
+    dsl
+  end
+
+  # DSL class for defining JMeter test plans.
+  # Provides methods to generate, save, and run JMeter tests.
+  class ExtendedDSL < DSL
+    include JmeterPerf::Helpers::Parser
+    attr_accessor :root
+
+    # Initializes an ExtendedDSL object with the provided parameters.
+    # @param params [Hash] Parameters for the test plan (default: `{}`).
+    def initialize(params = {})
+      @root = Nokogiri::XML(JmeterPerf::Helpers::String.strip_heredoc(
+        <<-EOF
+          <?xml version="1.0" encoding="UTF-8"?>
+          <jmeterTestPlan version="1.2" properties="3.1" jmeter="3.1" ruby-jmeter="3.0">
+          <hashTree>
+          </hashTree>
+          </jmeterTestPlan>
+        EOF
+      ))
+      node = JmeterPerf::DSL::TestPlan.new(params)
+      @current_node = @root.at_xpath("//jmeterTestPlan/hashTree")
+      @current_node = attach_to_last(node)
+    end
+
+    # Saves the test plan as a JMX file.
+    #
+    # @param out_jmx [String] The path for the output JMX file (default: `"ruby-jmeter.jmx"`).
+    def jmx(out_jmx: "ruby-jmeter.jmx")
+      File.write(out_jmx, doc.to_xml(indent: 2))
+      logger.info "JMX saved to: #{out_jmx}"
+    end
+
+    # Runs the test plan with the specified configuration.
+    #
+    # @param name [String] The name of the test run (default: `"ruby-jmeter"`).
+    # @param jmeter_path [String] Path to the JMeter executable (default: `"jmeter"`).
+    # @param out_jmx [String] The filename for the output JMX file (default: `"ruby-jmeter.jmx"`).
+    # @param out_jtl [String] The filename for the output JTL file (default: `"jmeter.jtl"`).
+    # @param out_jmeter_log [String] The filename for the JMeter log file (default: `"jmeter.log"`).
+    # @param out_cmd_log [String] The filename for the command log file (default: `"jmeter-cmd.log"`).
+    # @param jtl_read_timeout [Integer] The maximum number of seconds to wait for a line read (default: `3`).
+    # @return [JmeterPerf::Report::Summary] The summary report of the test run.
+    # @raise [RuntimeError] If the test execution fails.
+    def run(
+      name: "ruby-jmeter",
+      jmeter_path: "jmeter",
+      out_jmx: "ruby-jmeter.jmx",
+      out_jtl: "jmeter.jtl",
+      out_jmeter_log: "jmeter.log",
+      out_cmd_log: "jmeter-cmd.log",
+      jtl_read_timeout: 3
+    )
+      jmx(out_jmx:)
+      logger.warn "Executing #{out_jmx} test plan locally ..."
+
+      cmd = <<~CMD.strip
+        #{jmeter_path} -n -t #{out_jmx} -j #{out_jmeter_log} -l #{out_jtl}
+      CMD
+
+      summary = JmeterPerf::Report::Summary.new(file_path: out_jtl, name:, jtl_read_timeout:)
+      jtl_process_thread = summary.stream_jtl_async
+
+      File.open(out_cmd_log, "w") do |f|
+        pid = Process.spawn(cmd, out: f, err: [:child, :out])
+        Process.waitpid(pid)
+      end
+
+      summary.finish!         # Notify jtl collection that JTL cmd finished
+      jtl_process_thread.join # Join main thread and wait for it to finish
+
+      unless $?.exitstatus.zero?
+        logger.error("Failed to run #{cmd}. See #{out_cmd_log} and #{out_jmeter_log} for details.")
+        raise "Exit status: #{$?.exitstatus}"
+      end
+
+      summary.summarize_data!
+      logger.info "[Test Plan Execution Completed Successfully] JTL saved to: #{out_jtl}\n"
+      summary
+    rescue
+      summary.finish!
+      raise
+    end
+
+    private
+
+    # Creates a new hash tree node for the JMeter test plan.
+    #
+    # @return [Nokogiri::XML::Node] A new hash tree node.
+    def hash_tree
+      Nokogiri::XML::Node.new("hashTree", @root)
+    end
+
+    # Attaches a node as the last child of the current node.
+    #
+    # @param node [Object] The node to attach.
+    # @return [Object] The hash tree for the attached node.
+    def attach_to_last(node)
+      ht = hash_tree
+      last_node = @current_node
+      last_node << node.doc.children << ht
+      ht
+    end
+
+    # Attaches a node and evaluates the block within its context.
+    #
+    # @param node [Object] The node to attach.
+    # @yield The block to be executed in the node's context.
+    def attach_node(node, &block)
+      ht = attach_to_last(node)
+      previous = @current_node
+      @current_node = ht
+      instance_exec(&block) if block
+      @current_node = previous
+    end
+
+    # Returns the Nokogiri XML document.
+    #
+    # @return [Nokogiri::XML::Document] The XML document for the JMeter test plan.
+    def doc
+      Nokogiri::XML(@root.to_s, &:noblanks)
+    end
+
+    # Logger instance to log messages to the standard output.
+    #
+    # @return [Logger] The logger instance, initialized to DEBUG level.
+    def logger
+      return @logger if @logger
+      @logger = Logger.new($stdout)
+      @logger.level = Logger::DEBUG
+      @logger
+    end
+  end
+end