select_rails_log 0.2.0

data/lib/select_rails_log/printer/statistics_printer.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require "enumerable/statistics"
+
+module SelectRailsLog
+  module Printer
+    class StatisticsPrinter < BasePrinter
+      STAT_TOTAL_DURATION = "Total"
+
+      DEFAULT_TARGETS = [
+        STAT_TOTAL_DURATION,
+        PERFORMANCE_ACTIVE_RECORD,
+        PERFORMANCE_VIEWS,
+        PERFORMANCE_ALLOCATIONS
+      ].freeze
+
+      PERCENTILES = [25, 50, 75, 90, 95, 99].freeze
+
+      define_options :statistics_printer do
+        option :output, "--stats [FILE]", "-s", "Output statistics in TSV format", default: DEFAULT_OUTPUT
+        option :stats_targets,
+               "--stats-targets TARGETs",
+               "  Statistics targets", Array,
+               "    target can be one of #{DEFAULT_TARGETS.join(", ")}, or etc."
+      end
+
+      def initialize(*)
+        super
+
+        @stats_data = {}
+        @stats_targets = options[:stats_targets] || DEFAULT_TARGETS
+      end
+
+      def close
+        print_statistics
+        super
+      end
+
+      private
+
+      def init_output_destination
+        super
+        return unless output_directory?
+
+        raise CommandLineOptionError, "output to directory is not supported for statistics"
+      end
+
+      def build_stat_data
+        @stats_targets.each_with_object({}) do |target, hash|
+          hash[target] = []
+        end
+      end
+
+      def print_data(_output, data)
+        stat_data = @stats_data[data.values_at(CONTROLLER, ACTION)] ||= build_stat_data
+        stat_data.each do |target, values|
+          value = if target == STAT_TOTAL_DURATION
+                    data[DURATION]
+                  else
+                    data[PERFORMANCE][target]
+                  end
+          values << value if value
+        end
+      end
+
+      def print_statistics
+        print_row ["percentile", *PERCENTILES]
+
+        @stats_data.keys.sort.each do |(controller, action)|
+          @stats_data[[controller, action]].each do |target, values|
+            next if values.empty?
+
+            print_row ["#{controller}##{action} #{target}",
+                       *PERCENTILES.map { |percentile| values.percentile(percentile) }]
+          end
+        end
+      end
+
+      def print_row(values)
+        @output_file.puts values.join("\t")
+      end
+    end
+  end
+end

data/lib/select_rails_log/printer/text_printer.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module SelectRailsLog
+  module Printer
+    class TextPrinter < BasePrinter
+      SUFFIX = ".txt"
+      DATETIME_FORMAT = "%FT%T.%6N"
+
+      define_options :text_printer do
+        option :output, "--text [PATH]", "Output in text format (default)", default: DEFAULT_OUTPUT
+      end
+
+      def initialize(options, standard_output, fallback_output: nil)
+        @fallback_output = fallback_output
+        @first = true
+        super(options, standard_output)
+      end
+
+      private
+
+      def output_option
+        options[:output] || @fallback_output
+      end
+
+      def print_data(output, data)
+        output.puts unless @first || output_directory?
+        @first = false if @first
+
+        print_header(output, data)
+        print_body(output, data)
+      end
+
+      def print_header(output, data)
+        output.puts "time: #{data[STARTED]&.strftime(DATETIME_FORMAT)} " \
+                    ".. #{data[COMPLETED]&.strftime(DATETIME_FORMAT)}"
+        output.puts "request_id: #{data[REQUEST_ID]}" if data[REQUEST_ID]
+        output.print <<~END_OF_HEADER
+          pid: #{data[PID]}
+          status: #{data[HTTP_STATUS]}
+          duration: #{data[DURATION]}ms
+        END_OF_HEADER
+      end
+
+      def print_body(output, data)
+        each_log(data) do |log|
+          output.printf "[%<interval>8.3f] %<message>s\n", interval: log[INTERVAL] * 1_000, message: log[MESSAGE]
+        end
+      end
+    end
+  end
+end

data/lib/select_rails_log/printer/tsv_printer.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module SelectRailsLog
+  module Printer
+    class TsvPrinter < BasePrinter
+      DATETIME_FORMAT = "%F %T.%6N"
+
+      COLUMNS = [
+        STARTED, REQUEST_ID, "controller_action",
+        HTTP_STATUS, HTTP_METHOD, PATH,
+        "total_duration",
+        "active_record_duration",
+        "views_duration",
+        "allocations"
+      ].freeze
+
+      define_options :tsv_printer do
+        option :output, "--tsv [FILE]", "-t", "Output in TSV format", default: DEFAULT_OUTPUT
+      end
+
+      private
+
+      def init_output_destination
+        super
+        return unless output_directory?
+
+        raise CommandLineOptionError, "output to directory is not supported for TSV format"
+      end
+
+      def prepare
+        super
+        @csv = CSV.new(@output_file, col_sep: "\t", write_headers: true, headers: COLUMNS)
+      end
+
+      def print_data(_output, data)
+        @csv << row(data)
+      end
+
+      def row(data)
+        [
+          data[STARTED].strftime(DATETIME_FORMAT),
+          data[REQUEST_ID], "#{data[CONTROLLER]}##{data[ACTION]}",
+          data[HTTP_STATUS], data[HTTP_METHOD], data[PATH],
+          data[DURATION],
+          data[PERFORMANCE][PERFORMANCE_ACTIVE_RECORD],
+          data[PERFORMANCE][PERFORMANCE_VIEWS],
+          data[PERFORMANCE][PERFORMANCE_ALLOCATIONS]
+        ]
+      end
+    end
+  end
+end

data/lib/select_rails_log/printer.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require_relative "printer/base_printer"
+require_relative "printer/text_printer"
+require_relative "printer/raw_printer"
+require_relative "printer/json_printer"
+require_relative "printer/jsonl_printer"
+require_relative "printer/tsv_printer"
+require_relative "printer/statistics_printer"
+require_relative "printer/histgram_printer"
+require_relative "printer/boxplot_printer"
+require_relative "printer/null_printer"
+
+module SelectRailsLog
+  module Printer
+  end
+end

data/lib/select_rails_log/runner.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+module SelectRailsLog
+  class Runner < Extension
+    include Constants
+
+    define_options :runner do
+      separator ""
+      separator "other options:"
+
+      option :help, "--help", "-h", "Show help"
+      option :version, "--version", "Show version"
+      option :debug, "--debug", "Enable debug print"
+    end
+
+    class << self
+      def run(argv, argf, out = $stdout)
+        runner = setup_runner(argv, out)
+
+        begin
+          runner.run(Scanner.new(argf)) if runner.runnable?
+        rescue StopIteration, Errno::EPIPE, Interrupt
+          # noop
+        end
+
+        runner.success?
+      rescue StandardError => e
+        raise e if runner&.debug?
+
+        warn e.message
+        false
+      end
+
+      private
+
+      def setup_runner(argv, out)
+        options = CommandLineOptions.new
+        options.parse!(argv)
+
+        runner = Runner.new(options, out)
+        if runner.help?
+          out.puts options.parser
+        elsif runner.version?
+          print_version(out)
+        end
+
+        runner
+      end
+
+      def print_version(io)
+        io.print <<~VERSION
+          select_rails_log #{VERSION}
+           - csv #{CSV::VERSION}
+           - enumerable-statistics #{EnumerableStatistics::VERSION}
+           - unicode_plot #{UnicodePlot::VERSION}
+           - #{RUBY_ENGINE} #{RUBY_VERSION} [#{RUBY_PLATFORM}]
+        VERSION
+      end
+    end
+
+    def initialize(options, standard_output)
+      super(options)
+      @count = 0
+      @standard_output = standard_output
+    end
+
+    def help?
+      options[:help]
+    end
+
+    def version?
+      options[:version]
+    end
+
+    def debug?
+      options[:debug]
+    end
+
+    def success?
+      help? || version? || @count.positive?
+    end
+
+    def runnable?
+      !help? && !version?
+    end
+
+    def run(scanner)
+      selector, printers = setup
+      counter_thread = counter_thread() if output_tty?(printers)
+
+      keep_raw = include_raw_printer?(printers)
+      scanner.select(selector, keep_raw:) do |data|
+        @count += 1
+        printers.each { _1.print(data) }
+      end
+    ensure
+      counter_thread&.kill
+      printers&.each(&:close)
+    end
+
+    private
+
+    def output_tty?(printers)
+      @standard_output.tty? && printers.none?(&:output_stdout?)
+    end
+
+    def setup
+      filters = setup_filters
+      printers = setup_printers
+      printers << default_printer if printers.empty?
+      selector = Selector.new(filters)
+
+      [selector, printers]
+    end
+
+    def setup_filters
+      @whole_options.extensions(:filter).filter_map do |ext_class|
+        ext = ext_class.new(@whole_options)
+        ext if ext.runnable?
+      end
+    end
+
+    def setup_printers
+      @whole_options.extensions(:printer).filter_map do |ext_class|
+        ext = ext_class.new(@whole_options, @standard_output)
+        ext if ext.runnable?
+      end
+    end
+
+    def default_printer
+      Printer::TextPrinter.new(@whole_options, @standard_output, fallback_output: DEFAULT_OUTPUT)
+    end
+
+    def include_raw_printer?(printers)
+      printers.any? { _1.is_a?(Printer::RawPrinter) }
+    end
+
+    def counter_thread
+      Thread.new do
+        lambda do
+          loop do
+            sleep 1
+            print "\r#{@count}"
+          end
+        ensure
+          puts "\r#{@count}"
+        end
+      end
+    end
+  end
+end

data/lib/select_rails_log/scanner.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+require "time"
+
+module SelectRailsLog
+  class Scanner
+    include Constants
+
+    reqid_regexp = /\[(?<reqid>\h{8}-\h{4}-\h{4}-\h{4}-\h{12})\]/
+    LOG_REGEXP = /\A., \[(?<time>\S+) #(?<pid>\d+)\]  *(?<severity>\S+) -- :(?: #{reqid_regexp})? (?<message>.*)/
+    ANSI_ESCAPE_SEQ_REGEXP = /\e\[(?:\d{1,2}(?:;\d{1,2})?)?[mK]/
+
+    DATETIME_FORMAT = "%FT%T.%N"
+    REQUEST_FILTER_APPLIED = "request_filter_applied" # internal data keys
+    private_constant :DATETIME_FORMAT, :REQUEST_FILTER_APPLIED
+
+    def initialize(io)
+      @io = io
+    end
+
+    def select(selector, keep_raw: true)
+      buff = {}
+      prev_time = nil
+      prev_data = nil
+      found = false
+      stop_iteration = false
+
+      @io.each_line do |line|
+        m = LOG_REGEXP.match(line)
+        unless m
+          if prev_data && prev_data[LOGS].any?
+            prev_data[LOGS].last[MESSAGE] << "\n" << line.chomp.gsub(ANSI_ESCAPE_SEQ_REGEXP, "")
+            prev_data[RAW_LOGS].last << line if keep_raw
+          end
+          next
+        end
+
+        begin
+          time = Time.strptime(m[:time], DATETIME_FORMAT)
+        rescue ArgumentError
+          # ignore invalid time format
+        end
+
+        pid = m[:pid]
+        reqid = m[:reqid]
+        message = m[:message]
+        log = {
+          TIME => time,
+          MESSAGE => message,
+          SEVERITY => m[:severity]
+        }
+
+        ident = reqid || pid
+        data = prev_data = buff[ident] if buff.key?(ident)
+
+        if /\AStarted (?<http_method>\S+) "(?<path>[^"]*)" for (?<client>\S+)/ =~ message
+          buff.delete(ident)
+          if stop_iteration
+            prev_data = nil
+            buff.empty? ? break : next
+          end
+
+          log[INTERVAL] = 0.0
+          prev_time = time
+
+          data = {
+            ID => reqid || time.strftime("%Y%m%d-%H%M%S-%6N-#{pid}"),
+            STARTED => time,
+            PID => pid,
+            REQUEST_ID => reqid,
+            HTTP_METHOD => http_method,
+            PATH => path,
+            CLIENT => client,
+            LOGS => [log],
+            REQUEST_FILTER_APPLIED => false
+          }
+          data[RAW_LOGS] = [line] if keep_raw
+          buff[ident] = data
+          next
+        end
+
+        unless data
+          prev_data = nil
+          next
+        end
+
+        message.gsub!(ANSI_ESCAPE_SEQ_REGEXP, "")
+        log[INTERVAL] = (time && prev_time) ? time - prev_time : 0.0
+        prev_time = time
+
+        if /\AProcessing by (?<controller>[^\s#]+)#(?<action>\S+)/ =~ message
+          data[CONTROLLER] = controller
+          data[ACTION] = action
+          data[LOGS] << log
+          data[RAW_LOGS] << line if keep_raw
+
+          data.delete(REQUEST_FILTER_APPLIED)
+          begin
+            reqf_result = selector.run_request_filters(data)
+          rescue StopIteration
+            stop_iteration = true
+          end
+          if !reqf_result || stop_iteration
+            buff.delete(ident)
+            prev_data = nil
+          end
+        elsif /\A  Parameters: (?<params>.*)/ =~ message
+          data[PARAMETERS] = params
+          data[LOGS] << log
+          data[RAW_LOGS] << line if keep_raw
+        elsif /\ACompleted (?<http_status>\d+) .* in (?<duration>\d+)ms \((?<durations>.*)\)/ =~ message
+          data[HTTP_STATUS] = http_status
+          data[DURATION] = duration.to_i
+          data[PERFORMANCE] = durations.scan(/(\S+): (\d+(\.\d+)?)/)
+                                       .to_h { |type, dur, dur_f| [type, dur_f ? dur.to_f : dur.to_i] }
+          data[COMPLETED] = time
+          data[LOGS] << log
+          data[RAW_LOGS] << line if keep_raw
+
+          if data.key?(REQUEST_FILTER_APPLIED)
+            data.delete(REQUEST_FILTER_APPLIED)
+            reqf_result = selector.run_request_filters(data)
+          else
+            reqf_result = true
+          end
+
+          reqf_result && selector.run_line_filters(data) do |i|
+            yield(i)
+            found = true
+          end
+          buff.delete(ident)
+          prev_data = nil
+        else
+          data[LOGS] << log
+          data[RAW_LOGS] << line if keep_raw
+        end
+      end
+
+      found
+    end
+  end
+end

data/lib/select_rails_log/selector.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module SelectRailsLog
+  class Selector
+    def initialize(filters)
+      @request_filters = filters.select(&:request_filter?)
+      @line_filters = filters - @request_filters
+    end
+
+    def run_request_filters(data)
+      run_filters(data, @request_filters)
+    end
+
+    def run_line_filters(data, &)
+      run_filters(data, @line_filters, &)
+    end
+
+    private
+
+    def run_filters(data, filters)
+      result = if filters.empty?
+                 true
+               else
+                 filters.all? { |filter| filter.run(data) }
+               end
+
+      return yield(data) if result && block_given?
+
+      result
+    end
+  end
+end

data/lib/select_rails_log/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module SelectRailsLog
+  VERSION = "0.2.0"
+end

data/lib/select_rails_log.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require_relative "select_rails_log/constants"
+require_relative "select_rails_log/command_line_options"
+require_relative "select_rails_log/extension"
+require_relative "select_rails_log/filter"
+require_relative "select_rails_log/printer"
+require_relative "select_rails_log/selector"
+require_relative "select_rails_log/runner"
+require_relative "select_rails_log/scanner"
+require_relative "select_rails_log/version"
+
+module SelectRailsLog
+  class Error < RuntimeError; end
+  class CommandLineOptionError < Error; end
+
+  class << self
+    def run
+      exit(Runner.run(ARGV, ARGF) ? 0 : 1)
+    end
+  end
+end

data/sig/select_rails_log.rbs

@@ -0,0 +1,4 @@
+module SelectRailsLog
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end

metadata

@@ -0,0 +1,131 @@
+--- !ruby/object:Gem::Specification
+name: select_rails_log
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+platform: ruby
+authors:
+- akira yamada
+bindir: exe
+cert_chain: []
+date: 2025-01-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: csv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: enumerable-statistics
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: unicode_plot
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: select_rails_log is a tool for extracting request logs from Rails log
+  files.
+email:
+- akira@arika.org
+executables:
+- select_rails_log
+extensions: []
+extra_rdoc_files: []
+files:
+- ".rubocop.yml"
+- CHANGELOG.md
+- CODE_OF_CONDUCT.md
+- LICENSE.txt
+- README.md
+- Rakefile
+- exe/select_rails_log
+- images/boxplot.png
+- images/histgram.png
+- images/text.png
+- lib/select_rails_log.rb
+- lib/select_rails_log/command_line_options.rb
+- lib/select_rails_log/constants.rb
+- lib/select_rails_log/extension.rb
+- lib/select_rails_log/filter.rb
+- lib/select_rails_log/filter/base_filter.rb
+- lib/select_rails_log/filter/controller_action_filter.rb
+- lib/select_rails_log/filter/duration_range_filter.rb
+- lib/select_rails_log/filter/http_method_filter.rb
+- lib/select_rails_log/filter/http_status_filter.rb
+- lib/select_rails_log/filter/logs_regexp_filter.rb
+- lib/select_rails_log/filter/params_regexp_filter.rb
+- lib/select_rails_log/filter/range_pattern.rb
+- lib/select_rails_log/filter/request_id_filter.rb
+- lib/select_rails_log/filter/time_range_filter.rb
+- lib/select_rails_log/options.rb
+- lib/select_rails_log/printer.rb
+- lib/select_rails_log/printer/base_printer.rb
+- lib/select_rails_log/printer/boxplot_printer.rb
+- lib/select_rails_log/printer/data_serializable.rb
+- lib/select_rails_log/printer/histgram_printer.rb
+- lib/select_rails_log/printer/json_printer.rb
+- lib/select_rails_log/printer/jsonl_printer.rb
+- lib/select_rails_log/printer/null_printer.rb
+- lib/select_rails_log/printer/raw_printer.rb
+- lib/select_rails_log/printer/statistics_printer.rb
+- lib/select_rails_log/printer/text_printer.rb
+- lib/select_rails_log/printer/tsv_printer.rb
+- lib/select_rails_log/runner.rb
+- lib/select_rails_log/scanner.rb
+- lib/select_rails_log/selector.rb
+- lib/select_rails_log/version.rb
+- sig/select_rails_log.rbs
+homepage: https://github.com/arika/select_rails_log
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/arika/select_rails_log
+  source_code_uri: https://github.com/arika/select_rails_log
+  changelog_uri: https://github.com/arika/select_rails_log/blob/main/CHANGELOG.md
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.1.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.2
+specification_version: 4
+summary: Rails log selector
+test_files: []