select_rails_log 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 2e895ebdfdf2cfb2a09dc6771092b2df4128994657b888ca2f1c9fa0abd8e707
+  data.tar.gz: 3440ad80cde5de22eae7888543c7db28338e669ff5461b360afa762d7477c3d9
+SHA512:
+  metadata.gz: 4dc6fc31f2398e95f191af9b9704ae542ff60e71dd03dafa8fd93bd3f563b11d78ce9293e288319b6ecd66b11452278fb9e72ab868f3507bf6f085a58d8c3a91
+  data.tar.gz: 54cb6c4879c49ab1196a089e8fbd4632b72e3c2c5077c4c1918b5d8365c105d18a27d2334068fca8024e249f8df9c4a7d4e41f8a2a9b89e238ab8cb396f9606b

data/.rubocop.yml

@@ -0,0 +1,13 @@
+require:
+  - rubocop-performance
+  - rubocop-rake
+
+AllCops:
+  TargetRubyVersion: 3.1
+  NewCops: enable
+
+Style/StringLiterals:
+  EnforcedStyle: double_quotes
+
+Style/StringLiteralsInInterpolation:
+  EnforcedStyle: double_quotes

data/CHANGELOG.md

@@ -0,0 +1,22 @@
+## [Unreleased]
+
+## [0.2.1] - 2025-01-18
+
+- Multline log support
+- Bug fix
+- Refine stats
+- Refectoring
+- Add tests
+
+## [0.2.0] - 2024-12-15
+
+- Multiple printer support
+- Add printers
+  - jsonl
+  - tsv
+  - stats
+  - plot
+
+## [0.1.0] - 2024-12-12
+
+- Import from gists

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,132 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, caste, color, religion, or sexual
+identity and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the overall
+  community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or advances of
+  any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email address,
+  without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official email address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+[INSERT CONTACT METHOD].
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series of
+actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or permanent
+ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior, harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within the
+community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.1, available at
+[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
+
+Community Impact Guidelines were inspired by
+[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
+
+For answers to common questions about this code of conduct, see the FAQ at
+[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
+[https://www.contributor-covenant.org/translations][translations].
+
+[homepage]: https://www.contributor-covenant.org
+[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
+[Mozilla CoC]: https://github.com/mozilla/diversity
+[FAQ]: https://www.contributor-covenant.org/faq
+[translations]: https://www.contributor-covenant.org/translations

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2024 akira yamada
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,93 @@
+# select_rails_log
+
+select_rails_log is a tool for extracting request logs from Rails log files.
+
+It can be used during development to identify bottlenecks in the application or for small-scale benchmarking.
+
+The following extraction criteria can be specified:
+
+* Request ID
+* Controller name and action name
+* HTTP method and status code
+* Request date and time range
+* Response time range
+* String matching
+
+The extracted logs can be formatted and output in the following formats:
+
+* Text
+* JSON, JSONL
+* TSV
+* Raw log
+
+Additionally, the tool can aggregate and output metrics such as response times for the extracted requests:
+
+* Percentiles by controller and action
+* Box plots by controller and action
+* Histograms for the overall extraction results
+
+## Installation
+
+Install the gem by executing:
+
+```bash
+gem install select_rails_log
+```
+
+## Usage
+
+select_rails_log can process logs that include timestamps and Request-IDs.
+
+To enable this, first configure the appropriate settings file, such as `config/environments/development.rb`, as follows:
+
+```ruby
+config.log_tags = [:request_id]
+config.log_formatter = ::Logger::Formatter.new
+```
+
+### Sample session
+
+Here are some usage examples.
+
+To output a box plot for an overview, use the following command:
+
+```bash
+select_rails_log log/development.log --boxplot
+```
+
+The result will be output as follows:
+
+![boxplot](images/boxplot.png)
+
+Next, let’s examine the response time distribution for ActiveStorage::Representations::RedirectController#show, which had significant variance:
+
+```bash
+select_rails_log log/development.log -A 'ActiveStorage::Representations::RedirectController#show' --histgram
+```
+![histgram](images/histgram.png)
+
+Finally, let’s take a closer look at requests that took particularly long, specifically those over 400ms:
+
+```bash
+select_rails_log log/development.log -A 'ActiveStorage::Representations::RedirectController#show' -D 400..
+```
+
+![text](images/text.png)
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test-unit` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/arika/select_rails_log. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/arika/select_rails_log/blob/master/CODE_OF_CONDUCT.md).
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+## Code of Conduct
+
+Everyone interacting in the SelectRailsLog project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/arika/select_rails_log/blob/master/CODE_OF_CONDUCT.md).

data/Rakefile

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/*_test.rb"]
+end
+
+require "rubocop/rake_task"
+
+RuboCop::RakeTask.new
+
+task default: %i[test rubocop]

data/exe/select_rails_log

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "select_rails_log"
+SelectRailsLog.run

data/lib/select_rails_log/command_line_options.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require "optparse"
+require_relative "options"
+
+module SelectRailsLog
+  class CommandLineOptions < Options
+    class << self
+      attr_reader :initializers
+    end
+
+    attr_reader :parser
+
+    def initialize
+      super
+      @parser = OptionParser.new
+      @parser.banner += " [logfiles...]"
+      @extensions = {}
+      @extension_groups = {}
+      setup
+    end
+
+    def parse!(argv)
+      parser.parse!(argv)
+    end
+
+    def register(ext_name)
+      return if key?(ext_name)
+
+      self[ext_name] = Options.new
+    end
+
+    def add_option(ext_name, opt_name, *optparse_args, default: nil)
+      self[ext_name][opt_name] = nil
+
+      parser.on(*optparse_args) do |value|
+        value = default if value.nil?
+        self[ext_name][opt_name] = value
+      end
+    end
+
+    def extensions(group)
+      @extension_groups[group]
+    end
+
+    private
+
+    def setup
+      @extension_groups = {}
+      Extension.extensions.each do |ext_class|
+        ext_type = extension_type(ext_class)
+        @extension_groups[ext_type] ||= []
+        @extension_groups[ext_type] << ext_class
+
+        ext_class.option_initializers.each do |initializer|
+          initializer.call(self)
+        end
+      end
+    end
+
+    def extension_type(ext_class)
+      if ext_class < Filter::BaseFilter
+        :filter
+      elsif ext_class < Printer::BasePrinter
+        :printer
+      else
+        :other
+      end
+    end
+  end
+end

data/lib/select_rails_log/constants.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require "time"
+
+module SelectRailsLog
+  module Constants
+    DEFAULT_OUTPUT = Object.new
+
+    DEBUG = "DEBUG"
+
+    # data keys
+    ACTION = "action"
+    CLIENT = "client"
+    COMPLETED = "completed"
+    CONTROLLER = "controller"
+    DURATION = "duration"
+    HTTP_METHOD = "http_method"
+    HTTP_STATUS = "http_status"
+    ID = "id"
+    INTERVAL = "interval"
+    LOGS = "logs"
+    MESSAGE = "message"
+    PARAMETERS = "parameters"
+    PATH = "path"
+    PERFORMANCE = "performance"
+    PERFORMANCE_ACTIVE_RECORD = "ActiveRecord"
+    PERFORMANCE_ALLOCATIONS = "Allocations"
+    PERFORMANCE_VIEWS = "Views"
+    PID = "pid"
+    REQUEST_ID = "request_id"
+    RAW_LOGS = "raw_logs"
+    SEVERITY = "severity"
+    STARTED = "started"
+    TIME = "time"
+  end
+end

data/lib/select_rails_log/extension.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module SelectRailsLog
+  class Extension
+    @extensions = []
+
+    class << self
+      attr_reader :extensions, :extension_name
+
+      def option_initializers
+        @option_initializers ||= []
+      end
+
+      private
+
+      def option_initializer(&block)
+        option_initializers << block
+      end
+
+      def define_options(ext_name)
+        raise ArgumentError, "Extension #{ext_name} is already defined" if @extension_name
+
+        option_initializer do |opts|
+          opts.register(ext_name)
+        end
+
+        Extension.extensions << self
+        @extension_name = ext_name
+
+        yield
+      end
+
+      def option(opt_name, *optparse_args, default: nil)
+        ext_name = @extension_name
+
+        option_initializer do |opts|
+          opts.add_option(ext_name, opt_name, *optparse_args, default:)
+        end
+      end
+
+      def separator(str)
+        option_initializer do |opts|
+          opts.parser.separator(str)
+        end
+      end
+    end
+
+    attr_reader :options
+
+    def initialize(whole_options)
+      @whole_options = whole_options
+      @options = @whole_options[self.class.extension_name]
+    end
+
+    def runnable?
+      false
+    end
+  end
+end

data/lib/select_rails_log/filter/base_filter.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module SelectRailsLog
+  module Filter
+    class BaseFilter < Extension
+      include Constants
+
+      class << self
+        def filter_type(filter_type = nil)
+          @filter_type ||= :line
+          return @filter_type unless filter_type
+
+          @filter_type = filter_type
+        end
+      end
+
+      define_options :base_filter do
+        separator ""
+        separator "filter options:"
+      end
+
+      def request_filter?
+        self.class.filter_type == :request
+      end
+
+      def line_filter?
+        self.class.filter_type == :line
+      end
+
+      def run(obj)
+        obj
+      end
+    end
+  end
+end

data/lib/select_rails_log/filter/controller_action_filter.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module SelectRailsLog
+  module Filter
+    class ControllerActionFilter < BaseFilter
+      filter_type :request
+
+      define_options :controller_action_filter do
+        option :pattern,
+               "--controller-action NAMEs", "-A", Array,
+               "Filter by controller and action names",
+               "  ex: 'FooController#index,BarController,baz#show'"
+      end
+
+      def initialize(...)
+        super
+        @controller_actions = controller_actions(options[:pattern])
+      end
+
+      def runnable?
+        !!@controller_actions
+      end
+
+      def run(data)
+        @controller_actions.any? do |controller, action|
+          data[CONTROLLER] == controller &&
+            (action.nil? || data[ACTION] == action)
+        end
+      end
+
+      private
+
+      def controller_actions(names)
+        return unless names
+
+        names.map do |name|
+          controller, action = name.split("#", 2)
+          controller = classify(controller) << "Controller" unless controller.end_with?("Controller")
+          [controller, action]
+        end
+      end
+
+      def classify(name)
+        name.scan(%r{(?:/|[^_/]+)})
+            .map { |seg| seg == "/" ? "::" : seg.capitalize }
+            .join
+      end
+    end
+  end
+end

data/lib/select_rails_log/filter/duration_range_filter.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require_relative "range_pattern"
+
+module SelectRailsLog
+  module Filter
+    class DurationRangeFilter < BaseFilter
+      include RangePattern
+
+      define_options :duration_range_filter do
+        option :pattern,
+               "--duration-range RANGE", "-D", String,
+               "Filter by duration range [ms]",
+               "  range format is 'ms1..ms2', 'ms1...ms2', or 'ms,delta'.",
+               "  ex: '10..200', '10...200', '100,10'"
+      end
+
+      def initialize(...)
+        super
+
+        pattern = options[:pattern]
+        return unless pattern
+
+        begin
+          @range = parse_range_pattern(pattern, &:to_i)
+        rescue ArgumentError
+          raise CommandLineOptionError, "invalid duration range pattern `#{pattern}`"
+        end
+      end
+
+      def runnable?
+        !!@range
+      end
+
+      def run(data)
+        @range.cover?(data[DURATION])
+      end
+    end
+  end
+end

data/lib/select_rails_log/filter/http_method_filter.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module SelectRailsLog
+  module Filter
+    class HttpMethodFilter < BaseFilter
+      filter_type :request
+
+      define_options :http_method_filter do
+        option :http_method, "--http-method METHOD", "-M", String, "Filter by HTTP method"
+      end
+
+      def initialize(...)
+        super
+        @http_method = options[:http_method]&.upcase
+      end
+
+      def runnable?
+        !!@http_method
+      end
+
+      def run(data)
+        data[HTTP_METHOD] == @http_method
+      end
+    end
+  end
+end

data/lib/select_rails_log/filter/http_status_filter.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module SelectRailsLog
+  module Filter
+    class HttpStatusFilter < BaseFilter
+      define_options :http_status_filter do
+        option :pattern,
+               "--http-statuses PATTERN", "-S", String,
+               "Filter by HTTP statuses",
+               "  statuses format is 'st1,st2,...', '!st1,st2,...', or 'st1,...!st2,...'.",
+               "  ex: '200,302', '3,20,!201,304', or '!4,5'"
+      end
+
+      def initialize(...)
+        super
+
+        @includes = @excludes = []
+        pattern = options[:pattern]
+        return unless pattern
+
+        m = /\A([,\d]+)?(?:!([,\d]+))?\z/.match(pattern)
+        raise CommandLineOptionError, "invalid HTTP statuses pattern `#{pattern}`" unless m
+
+        @includes = m[1] ? m[1].split(",") : []
+        @excludes = m[2] ? m[2].split(",") : []
+      end
+
+      def runnable?
+        @includes.any? || @excludes.any?
+      end
+
+      def run(data)
+        http_status = data[HTTP_STATUS]
+        @excludes.none? { |exc| http_status.index(exc)&.zero? } &&
+          (@includes.empty? || @includes.any? { |inc| http_status.index(inc)&.zero? })
+      end
+    end
+  end
+end