onlylogs 0.7.0 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 19813b5a5884acae634a3516e4cd23dc67fb5cb359062491a15d331f35de2448
-  data.tar.gz: 158e9b56b481744b67a6d92b0835f1443f769b0f264149fb71fe1b69fc86e250
+  metadata.gz: f166264d89c5d2273751e3377ef236168da610b1d37d5ddf02d33959f2fe318d
+  data.tar.gz: a612d6738ee600171cd21a78b403d1abef06fdf5df893ebcc8c3401b092f6a51
 SHA512:
-  metadata.gz: 9b2c8cd637017af2c310b06a406d75c0a70659244f6b93dfcb12b0f470337d87c8254e257fa4e4810faea6b12e0f2ad27a60b40d2c3423cc98deb21cf56df39d
-  data.tar.gz: d2b9af11af98db9c2d92142553a1a7cc3eb915ddc398ed91b34a04d201e4cbda3dbc926bd007fef335071cbfc77e91ce22b3ff272c3eb7ffa8b607b032841cc6
+  metadata.gz: 6c2d38d16486ea48cd8e6c695e1b9234dbe8e300f45342afa163adfef2e532cdf1cf0cdecdca411c9a13a80c6c4f32aa5c33a82dfc9137c2da35f72cffa0f2c6
+  data.tar.gz: d4b8b016e5c089712e288d6533b613571f5edee524b2784a3aff80b4d8701913aabb169f72cdedb4220c01a86f3be33d0c52eedc644538db1a3fe3e259e947c9

data/README.md

@@ -272,6 +272,22 @@ Onlylogs.configure do |config|
 end
 ```
 
+#### Bounding How Long a Viewer Search Can Run
+
+Searches started from the log viewer stop after `search_timeout` seconds, keeping whatever they
+found and reporting that they did not reach the end of the file. The default is 120 seconds:
+
+```ruby
+# config/initializers/onlylogs.rb
+Onlylogs.configure do |config|
+  config.search_timeout = 300
+
+  # Or remove the ceiling entirely (a search then holds a CPU until it reaches
+  # the end of the file, however large it is)
+  config.search_timeout = nil
+end
+```
+
 ### Filtering Log Lines with a Denylist
 
 The `Onlylogs::Formatter` supports a denylist: an array of regular expressions that prevents matching lines from being logged. This is useful for filtering out noisy or irrelevant entries like health checks or asset requests.

data/app/channels/onlylogs/logs_channel.rb

@@ -173,7 +173,8 @@ module Onlylogs
 
           if filter.present?
             # Use grep for filtered search
-            @log_file.grep(filter, regexp_mode: regexp_mode, start_position: start_position, end_position: end_position) do |result|
+            @log_file.grep(filter, regexp_mode: regexp_mode, start_position: start_position,
+              end_position: end_position, timeout: Onlylogs.search_timeout) do |result|
               break if @batch_sender.nil? || @log_watcher_running == false
 
               # Skip first line if start_position > 0 (line is cut off at byte boundary)
@@ -234,6 +235,11 @@ module Onlylogs
         else
           transmit({action: "finish", content: "Search finished."})
         end
+      rescue Onlylogs::Grep::TimeoutError
+        @batch_sender&.stop
+        transmit({action: "finish",
+                  content: "Search stopped after #{Onlylogs.search_timeout} seconds and did not reach " \
+                           "the end of the file. Narrow the range or the filter and search again."})
       ensure
         # Always cleanup even if interrupted or error occurs
         @batch_sender&.stop

data/app/models/onlylogs/file.rb

@@ -55,8 +55,12 @@ module Onlylogs
       true
     end
 
-    def grep(filter, regexp_mode: false, start_position: 0, end_position: nil, &block)
-      Grep.grep(filter, path, regexp_mode: regexp_mode, start_position: start_position, end_position: end_position) do |content|
+    # +timeout+ is in seconds and defaults to nil, meaning unbounded. See
+    # Grep.grep.
+    def grep(filter, regexp_mode: false, start_position: 0, end_position: nil,
+      max_matches: Onlylogs.max_line_matches, timeout: nil, &block)
+      Grep.grep(filter, path, regexp_mode: regexp_mode, start_position: start_position,
+        end_position: end_position, max_matches: max_matches, timeout: timeout) do |content|
         yield content
       end
     end

data/app/models/onlylogs/grep.rb

@@ -1,21 +1,21 @@
+require "timeout"
+
 module Onlylogs
   class Grep
-    def self.grep(pattern, file_path, start_position: 0, end_position: nil, regexp_mode: false, &block)
-      # Use the appropriate script based on configuration
-      script_name = Onlylogs.ripgrep_enabled? ? "super_ripgrep" : "super_grep"
-      super_grep_path = ::File.expand_path("../../../bin/#{script_name}", __dir__)
+    # Raised when a search runs past its own +timeout+. It inherits from
+    # Timeout::Error so callers that already wrap searches in Timeout.timeout
+    # can keep a single rescue.
+    class TimeoutError < ::Timeout::Error; end
 
-      command_args = [super_grep_path]
-      command_args += ["--max-matches", Onlylogs.max_line_matches.to_s] if Onlylogs.max_line_matches.present?
-      command_args << "--regexp" if regexp_mode
-
-      # Add byte range parameters if specified
-      if start_position > 0 || end_position
-        command_args << "--start-position" << start_position.to_s
-        command_args << "--end-position" << end_position.to_s if end_position
-      end
-
-      command_args += [pattern, file_path]
+    # +timeout+ is in seconds and defaults to nil, which is no deadline at all:
+    # the search runs until it finishes or the caller abandons it. The default
+    # is deliberately not a number, because only the caller knows how long it
+    # can afford to hold the thread it runs on. Anything serving a request
+    # should pass one.
+    def self.grep(pattern, file_path, start_position: 0, end_position: nil, regexp_mode: false,
+      max_matches: Onlylogs.max_line_matches, timeout: nil, &block)
+      command_args = search_command(pattern, file_path, start_position: start_position,
+        end_position: end_position, regexp_mode: regexp_mode, max_matches: max_matches)
 
       results = []
 
@@ -23,20 +23,25 @@ module Onlylogs
       parse_line = if Onlylogs.ripgrep_enabled?
         ->(line) {
           parts = line.split(":", 2)
-          [parts[0].to_i, parts[1] || ""]
+          [parts[0].to_i + start_position, parts[1] || ""]
         }
       else
         ->(line) { [nil, line] }
       end
 
-      IO.popen(command_args, err: "/dev/null") do |io|
-        io.each_line do |line|
+      matches = 0
+
+      ActiveSupport::Notifications.instrument("search.onlylogs", file_path: file_path,
+        query: pattern, regexp: regexp_mode, start_position: start_position,
+        end_position: end_position, max_matches: max_matches) do |payload|
+        each_output_line(command_args, timeout: timeout) do |line|
           byte_offset, content = parse_line.call(line.chomp)
 
           # Use String.new to create a copy and prevent memory retention from IO buffers
           content = String.new(content, encoding: Encoding::UTF_8).scrub
 
           result = {byte_offset: byte_offset, content: content}
+          matches += 1
 
           if block_given?
             yield result
@@ -44,19 +49,112 @@ module Onlylogs
             results << result
           end
         end
+      rescue TimeoutError
+        payload[:timed_out] = true
+        raise
       ensure
+        payload[:matches] = matches
+        payload[:timed_out] ||= false
         drop_page_cache(file_path)
       end
 
       block_given? ? nil : results
     end
 
+    def self.search_command(pattern, file_path, start_position: 0, end_position: nil, regexp_mode: false,
+      max_matches: Onlylogs.max_line_matches)
+      script_name = Onlylogs.ripgrep_enabled? ? "super_ripgrep" : "super_grep"
+      super_grep_path = ::File.expand_path("../../../bin/#{script_name}", __dir__)
+
+      command_args = [super_grep_path]
+      command_args += ["--max-matches", max_matches.to_s] if max_matches.present?
+      command_args << "--regexp" if regexp_mode
+
+      # Add byte range parameters if specified
+      if start_position > 0 || end_position
+        command_args << "--start-position" << start_position.to_s
+        command_args << "--end-position" << end_position.to_s if end_position
+      end
+
+      command_args + [pattern, file_path]
+    end
+
+    # Runs the search subprocess and yields its output line by line.
+    #
+    # The child is a shell pipeline (tail | head | rg) that can spend minutes
+    # scanning a multi-GB file, so two things have to hold. timeout(1) bounds
+    # the run and escalates TERM to KILL on the whole pipeline by itself, which
+    # covers the deadline. The rest is the caller walking away early - a break
+    # out of the yield, an exception, a dropped connection - and there the order
+    # below is the point: signal the process group *before* closing the pipe.
+    # Closing first waits for a child that, having matched nothing, never wrote
+    # and so never received SIGPIPE. That is how a 25 second timeout once turned
+    # into a 24 minute request.
+    def self.each_output_line(command_args, timeout: nil, &block)
+      reader, writer = IO.pipe
+
+      begin
+        pid = Process.spawn(*bounded(command_args, timeout), out: writer, err: ::File::NULL, pgroup: true)
+      rescue
+        reader.close
+        raise
+      ensure
+        writer.close
+      end
+
+      begin
+        reader.each_line(&block)
+      ensure
+        status = stop(pid)
+        reader.close
+      end
+
+      raise TimeoutError, "search exceeded #{timeout}s" if status&.exitstatus == TIMED_OUT_EXIT_STATUS
+    end
+
+    # timeout(1) exits with this when it had to stop the command.
+    TIMED_OUT_EXIT_STATUS = 124
+    KILL_GRACE_PERIOD = 0.5
+
+    def self.bounded(command_args, timeout)
+      return command_args unless timeout && timeout_command_available?
+
+      ["timeout", "-k", KILL_GRACE_PERIOD.to_s, timeout.to_s, *command_args]
+    end
+
+    # Not part of a BSD userland, so a machine without GNU coreutils falls back
+    # to whatever deadline the caller imposes. The kill path below still works.
+    def self.timeout_command_available?
+      return @timeout_command_available if defined?(@timeout_command_available)
+
+      @timeout_command_available = system("command -v timeout > /dev/null 2>&1")
+    end
+
+    # Runs while unwinding from an async exception often enough that a second
+    # one - another Timeout::Error, a Thread#kill from a shutting down server -
+    # could otherwise land between the signal and the reap and leave the
+    # pipeline running with nobody left to stop it.
+    def self.stop(pid)
+      Thread.handle_interrupt(::Exception => :never) do
+        begin
+          Process.kill("TERM", -pid)
+        rescue Errno::ESRCH, Errno::EPERM
+          nil
+        end
+
+        Process.waitpid2(pid).last
+      end
+    rescue Errno::ECHILD
+      nil
+    end
+
     # Searching a large log file pulls the whole file into the OS page cache.
     # In a container the kernel charges that cache to the cgroup, so a few
     # searches over multi-GB logs can exhaust the memory limit and trigger an
     # OOM kill even though no Ruby memory leaked. Hint the kernel to drop the
-    # pages we just read. Best-effort: advise is only a hint and is unsupported
-    # on some platforms, so never let it break a search.
+    # pages we just read, once the child is gone and nothing is refilling them.
+    # Best-effort: advise is only a hint and is unsupported on some platforms,
+    # so never let it break a search.
     def self.drop_page_cache(file_path)
       ::File.open(file_path) { |file| file.advise(:dontneed) }
     rescue

data/bin/super_grep

@@ -5,10 +5,6 @@ export LC_ALL=C
 max_matches=""
 start_position=""
 end_position=""
-# Block size for dd operations. Performance testing shows no significant
-# difference across block sizes (4K-8M) on typical log files. 1M balances
-# memory usage and I/O efficiency.
-block_size="${BLOCK_SIZE:-1M}"
 
 while [[ $# -gt 0 ]]; do
   case "$1" in
@@ -67,26 +63,22 @@ fi
 
 # Handle byte range if specified
 if [ -n "$start_position" ] || [ -n "$end_position" ]; then
+  case "${start_position:-0}${end_position:-0}" in
+    *[!0-9]*) exit 1 ;;
+  esac
+
   file_size=$(wc -c < "$file")
   range_start=${start_position:-0}
   range_end=${end_position:-$file_size}
   range_size=$((range_end - range_start))
-  
-  # Validate range
-  if [ $range_start -lt 0 ] || [ $range_size -le 0 ] || [ $range_start -ge $file_size ]; then
+
+  if [ $range_size -le 0 ] || [ $range_start -ge $file_size ]; then
     exit 0
   fi
-  
-  # Adjust if exceeds file size
+
   [ $range_end -gt $file_size ] && range_end=$file_size && range_size=$((range_end - range_start))
-  
-  # Extract byte range using dd
-  start_mb=$((range_start / 1048576))
-  start_offset=$((range_start % 1048576))
-  count_mb=$(((range_size + 1048576 - 1) / 1048576))
-  
-  dd if="$file" bs="$block_size" skip=$start_mb count=$count_mb 2>/dev/null | \
-    dd bs=1 skip=$start_offset count=$range_size 2>/dev/null | \
+
+  tail -c +$((range_start + 1)) "$file" | head -c $range_size | \
     "${grep_cmd[@]}" "$query_regex"
 else
   # Search entire file

data/bin/super_ripgrep

@@ -5,10 +5,6 @@ export LC_ALL=C
 max_matches=""
 start_position=""
 end_position=""
-# Block size for dd operations. Performance testing shows no significant
-# difference across block sizes (4K-8M) on typical log files. 1M balances
-# memory usage and I/O efficiency.
-block_size="${BLOCK_SIZE:-1M}"
 
 while [[ $# -gt 0 ]]; do
   case "$1" in
@@ -69,26 +65,22 @@ rg_cmd="rg --color=never --no-filename --byte-offset --no-mmap"
 
 # Handle byte range if specified
 if [ -n "$start_position" ] || [ -n "$end_position" ]; then
+  case "${start_position:-0}${end_position:-0}" in
+    *[!0-9]*) exit 1 ;;
+  esac
+
   file_size=$(wc -c < "$file")
   range_start=${start_position:-0}
   range_end=${end_position:-$file_size}
   range_size=$((range_end - range_start))
-  
-  # Validate range
-  if [ $range_start -lt 0 ] || [ $range_size -le 0 ] || [ $range_start -ge $file_size ]; then
+
+  if [ $range_size -le 0 ] || [ $range_start -ge $file_size ]; then
     exit 0
   fi
-  
-  # Adjust if exceeds file size
+
   [ $range_end -gt $file_size ] && range_end=$file_size && range_size=$((range_end - range_start))
-  
-  # Extract byte range using dd
-  start_mb=$((range_start / 1048576))
-  start_offset=$((range_start % 1048576))
-  count_mb=$(((range_size + 1048576 - 1) / 1048576))
-  
-  dd if="$file" bs="$block_size" skip=$start_mb count=$count_mb 2>/dev/null | \
-    dd bs=1 skip=$start_offset count=$range_size 2>/dev/null | \
+
+  tail -c +$((range_start + 1)) "$file" | head -c $range_size | \
     $rg_cmd -e "$query_regex"
 else
   # Search entire file

data/lib/onlylogs/configuration.rb

@@ -4,7 +4,14 @@ module Onlylogs
   class Configuration
     attr_accessor :log_file_patterns, :default_log_file_path, :basic_auth_user, :basic_auth_password,
       :parent_controller, :disable_basic_authentication, :ripgrep_enabled, :editor,
-      :max_line_matches
+      :max_line_matches, :search_timeout
+
+    # Seconds a viewer search may run before it is stopped. Bounded by default:
+    # a search holds a CPU for as long as it runs, and an unbounded one over a
+    # multi-GB file can starve everything else the host is doing. Generous
+    # enough for a full scan of a large file - set it to nil to remove the
+    # ceiling entirely.
+    DEFAULT_SEARCH_TIMEOUT = 120
 
     def initialize
       @log_file_patterns = default_log_file_patterns
@@ -16,6 +23,7 @@ module Onlylogs
       @ripgrep_enabled = default_ripgrep_enabled
       @editor = nil
       @max_line_matches = 100000
+      @search_timeout = DEFAULT_SEARCH_TIMEOUT
     end
 
     def configure
@@ -147,6 +155,10 @@ module Onlylogs
     configuration.max_line_matches
   end
 
+  def self.search_timeout
+    configuration.search_timeout
+  end
+
   def self.allowed_file_patterns_for(pattern)
     absolute_pattern = ::File.expand_path(pattern.to_s)
     if glob_pattern?(absolute_pattern)

data/lib/onlylogs/version.rb

@@ -1,3 +1,3 @@
 module Onlylogs
-  VERSION = "0.7.0"
+  VERSION = "0.9.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: onlylogs
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.9.0
 platform: ruby
 authors:
 - Alessandro Rodi