onlylogs 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 19813b5a5884acae634a3516e4cd23dc67fb5cb359062491a15d331f35de2448
-  data.tar.gz: 158e9b56b481744b67a6d92b0835f1443f769b0f264149fb71fe1b69fc86e250
+  metadata.gz: 0e6b4ee2fde339a248ff9e15557c8bb8f0ea525b3807aa09b157e537057c0c7a
+  data.tar.gz: 208543eb460dc3f7c302fac4d4bd3277c710e402346f759c7e66c1b2dfc7049b
 SHA512:
-  metadata.gz: 9b2c8cd637017af2c310b06a406d75c0a70659244f6b93dfcb12b0f470337d87c8254e257fa4e4810faea6b12e0f2ad27a60b40d2c3423cc98deb21cf56df39d
-  data.tar.gz: d2b9af11af98db9c2d92142553a1a7cc3eb915ddc398ed91b34a04d201e4cbda3dbc926bd007fef335071cbfc77e91ce22b3ff272c3eb7ffa8b607b032841cc6
+  metadata.gz: 4b47a580f758cacca9c34b1de0ebe445ff4db5df1e45f6e515cd80b1dd866d3dbbb39d7aea7862b94ee46e5d18cd39797fbadcb202e59745bcb94fb0a62da2b7
+  data.tar.gz: 8956f328a31a63cbf07a180a51ffa612c516f51ae37523bba49f6a41557a14ae35a83d91bc1bcc05b465bedd6cbc61cbffb8a402822a27b17d534668acd65211

data/app/models/onlylogs/file.rb

@@ -55,8 +55,12 @@ module Onlylogs
       true
     end
 
-    def grep(filter, regexp_mode: false, start_position: 0, end_position: nil, &block)
-      Grep.grep(filter, path, regexp_mode: regexp_mode, start_position: start_position, end_position: end_position) do |content|
+    # +timeout+ is in seconds and defaults to nil, meaning unbounded. See
+    # Grep.grep.
+    def grep(filter, regexp_mode: false, start_position: 0, end_position: nil,
+      max_matches: Onlylogs.max_line_matches, timeout: nil, &block)
+      Grep.grep(filter, path, regexp_mode: regexp_mode, start_position: start_position,
+        end_position: end_position, max_matches: max_matches, timeout: timeout) do |content|
         yield content
       end
     end

data/app/models/onlylogs/grep.rb

@@ -1,21 +1,21 @@
+require "timeout"
+
 module Onlylogs
   class Grep
-    def self.grep(pattern, file_path, start_position: 0, end_position: nil, regexp_mode: false, &block)
-      # Use the appropriate script based on configuration
-      script_name = Onlylogs.ripgrep_enabled? ? "super_ripgrep" : "super_grep"
-      super_grep_path = ::File.expand_path("../../../bin/#{script_name}", __dir__)
-
-      command_args = [super_grep_path]
-      command_args += ["--max-matches", Onlylogs.max_line_matches.to_s] if Onlylogs.max_line_matches.present?
-      command_args << "--regexp" if regexp_mode
-
-      # Add byte range parameters if specified
-      if start_position > 0 || end_position
-        command_args << "--start-position" << start_position.to_s
-        command_args << "--end-position" << end_position.to_s if end_position
-      end
+    # Raised when a search runs past its own +timeout+. It inherits from
+    # Timeout::Error so callers that already wrap searches in Timeout.timeout
+    # can keep a single rescue.
+    class TimeoutError < ::Timeout::Error; end
 
-      command_args += [pattern, file_path]
+    # +timeout+ is in seconds and defaults to nil, which is no deadline at all:
+    # the search runs until it finishes or the caller abandons it. The default
+    # is deliberately not a number, because only the caller knows how long it
+    # can afford to hold the thread it runs on. Anything serving a request
+    # should pass one.
+    def self.grep(pattern, file_path, start_position: 0, end_position: nil, regexp_mode: false,
+      max_matches: Onlylogs.max_line_matches, timeout: nil, &block)
+      command_args = search_command(pattern, file_path, start_position: start_position,
+        end_position: end_position, regexp_mode: regexp_mode, max_matches: max_matches)
 
       results = []
 
@@ -23,14 +23,14 @@ module Onlylogs
       parse_line = if Onlylogs.ripgrep_enabled?
         ->(line) {
           parts = line.split(":", 2)
-          [parts[0].to_i, parts[1] || ""]
+          [parts[0].to_i + start_position, parts[1] || ""]
         }
       else
         ->(line) { [nil, line] }
       end
 
-      IO.popen(command_args, err: "/dev/null") do |io|
-        io.each_line do |line|
+      begin
+        each_output_line(command_args, timeout: timeout) do |line|
           byte_offset, content = parse_line.call(line.chomp)
 
           # Use String.new to create a copy and prevent memory retention from IO buffers
@@ -51,12 +51,100 @@ module Onlylogs
       block_given? ? nil : results
     end
 
+    def self.search_command(pattern, file_path, start_position: 0, end_position: nil, regexp_mode: false,
+      max_matches: Onlylogs.max_line_matches)
+      script_name = Onlylogs.ripgrep_enabled? ? "super_ripgrep" : "super_grep"
+      super_grep_path = ::File.expand_path("../../../bin/#{script_name}", __dir__)
+
+      command_args = [super_grep_path]
+      command_args += ["--max-matches", max_matches.to_s] if max_matches.present?
+      command_args << "--regexp" if regexp_mode
+
+      # Add byte range parameters if specified
+      if start_position > 0 || end_position
+        command_args << "--start-position" << start_position.to_s
+        command_args << "--end-position" << end_position.to_s if end_position
+      end
+
+      command_args + [pattern, file_path]
+    end
+
+    # Runs the search subprocess and yields its output line by line.
+    #
+    # The child is a shell pipeline (tail | head | rg) that can spend minutes
+    # scanning a multi-GB file, so two things have to hold. timeout(1) bounds
+    # the run and escalates TERM to KILL on the whole pipeline by itself, which
+    # covers the deadline. The rest is the caller walking away early - a break
+    # out of the yield, an exception, a dropped connection - and there the order
+    # below is the point: signal the process group *before* closing the pipe.
+    # Closing first waits for a child that, having matched nothing, never wrote
+    # and so never received SIGPIPE. That is how a 25 second timeout once turned
+    # into a 24 minute request.
+    def self.each_output_line(command_args, timeout: nil, &block)
+      reader, writer = IO.pipe
+
+      begin
+        pid = Process.spawn(*bounded(command_args, timeout), out: writer, err: ::File::NULL, pgroup: true)
+      rescue
+        reader.close
+        raise
+      ensure
+        writer.close
+      end
+
+      begin
+        reader.each_line(&block)
+      ensure
+        status = stop(pid)
+        reader.close
+      end
+
+      raise TimeoutError, "search exceeded #{timeout}s" if status&.exitstatus == TIMED_OUT_EXIT_STATUS
+    end
+
+    # timeout(1) exits with this when it had to stop the command.
+    TIMED_OUT_EXIT_STATUS = 124
+    KILL_GRACE_PERIOD = 0.5
+
+    def self.bounded(command_args, timeout)
+      return command_args unless timeout && timeout_command_available?
+
+      ["timeout", "-k", KILL_GRACE_PERIOD.to_s, timeout.to_s, *command_args]
+    end
+
+    # Not part of a BSD userland, so a machine without GNU coreutils falls back
+    # to whatever deadline the caller imposes. The kill path below still works.
+    def self.timeout_command_available?
+      return @timeout_command_available if defined?(@timeout_command_available)
+
+      @timeout_command_available = system("command -v timeout > /dev/null 2>&1")
+    end
+
+    # Runs while unwinding from an async exception often enough that a second
+    # one - another Timeout::Error, a Thread#kill from a shutting down server -
+    # could otherwise land between the signal and the reap and leave the
+    # pipeline running with nobody left to stop it.
+    def self.stop(pid)
+      Thread.handle_interrupt(::Exception => :never) do
+        begin
+          Process.kill("TERM", -pid)
+        rescue Errno::ESRCH, Errno::EPERM
+          nil
+        end
+
+        Process.waitpid2(pid).last
+      end
+    rescue Errno::ECHILD
+      nil
+    end
+
     # Searching a large log file pulls the whole file into the OS page cache.
     # In a container the kernel charges that cache to the cgroup, so a few
     # searches over multi-GB logs can exhaust the memory limit and trigger an
     # OOM kill even though no Ruby memory leaked. Hint the kernel to drop the
-    # pages we just read. Best-effort: advise is only a hint and is unsupported
-    # on some platforms, so never let it break a search.
+    # pages we just read, once the child is gone and nothing is refilling them.
+    # Best-effort: advise is only a hint and is unsupported on some platforms,
+    # so never let it break a search.
     def self.drop_page_cache(file_path)
       ::File.open(file_path) { |file| file.advise(:dontneed) }
     rescue

data/bin/super_grep

@@ -5,10 +5,6 @@ export LC_ALL=C
 max_matches=""
 start_position=""
 end_position=""
-# Block size for dd operations. Performance testing shows no significant
-# difference across block sizes (4K-8M) on typical log files. 1M balances
-# memory usage and I/O efficiency.
-block_size="${BLOCK_SIZE:-1M}"
 
 while [[ $# -gt 0 ]]; do
   case "$1" in
@@ -67,26 +63,22 @@ fi
 
 # Handle byte range if specified
 if [ -n "$start_position" ] || [ -n "$end_position" ]; then
+  case "${start_position:-0}${end_position:-0}" in
+    *[!0-9]*) exit 1 ;;
+  esac
+
   file_size=$(wc -c < "$file")
   range_start=${start_position:-0}
   range_end=${end_position:-$file_size}
   range_size=$((range_end - range_start))
-  
-  # Validate range
-  if [ $range_start -lt 0 ] || [ $range_size -le 0 ] || [ $range_start -ge $file_size ]; then
+
+  if [ $range_size -le 0 ] || [ $range_start -ge $file_size ]; then
     exit 0
   fi
-  
-  # Adjust if exceeds file size
+
   [ $range_end -gt $file_size ] && range_end=$file_size && range_size=$((range_end - range_start))
-  
-  # Extract byte range using dd
-  start_mb=$((range_start / 1048576))
-  start_offset=$((range_start % 1048576))
-  count_mb=$(((range_size + 1048576 - 1) / 1048576))
-  
-  dd if="$file" bs="$block_size" skip=$start_mb count=$count_mb 2>/dev/null | \
-    dd bs=1 skip=$start_offset count=$range_size 2>/dev/null | \
+
+  tail -c +$((range_start + 1)) "$file" | head -c $range_size | \
     "${grep_cmd[@]}" "$query_regex"
 else
   # Search entire file

data/bin/super_ripgrep

@@ -5,10 +5,6 @@ export LC_ALL=C
 max_matches=""
 start_position=""
 end_position=""
-# Block size for dd operations. Performance testing shows no significant
-# difference across block sizes (4K-8M) on typical log files. 1M balances
-# memory usage and I/O efficiency.
-block_size="${BLOCK_SIZE:-1M}"
 
 while [[ $# -gt 0 ]]; do
   case "$1" in
@@ -69,26 +65,22 @@ rg_cmd="rg --color=never --no-filename --byte-offset --no-mmap"
 
 # Handle byte range if specified
 if [ -n "$start_position" ] || [ -n "$end_position" ]; then
+  case "${start_position:-0}${end_position:-0}" in
+    *[!0-9]*) exit 1 ;;
+  esac
+
   file_size=$(wc -c < "$file")
   range_start=${start_position:-0}
   range_end=${end_position:-$file_size}
   range_size=$((range_end - range_start))
-  
-  # Validate range
-  if [ $range_start -lt 0 ] || [ $range_size -le 0 ] || [ $range_start -ge $file_size ]; then
+
+  if [ $range_size -le 0 ] || [ $range_start -ge $file_size ]; then
     exit 0
   fi
-  
-  # Adjust if exceeds file size
+
   [ $range_end -gt $file_size ] && range_end=$file_size && range_size=$((range_end - range_start))
-  
-  # Extract byte range using dd
-  start_mb=$((range_start / 1048576))
-  start_offset=$((range_start % 1048576))
-  count_mb=$(((range_size + 1048576 - 1) / 1048576))
-  
-  dd if="$file" bs="$block_size" skip=$start_mb count=$count_mb 2>/dev/null | \
-    dd bs=1 skip=$start_offset count=$range_size 2>/dev/null | \
+
+  tail -c +$((range_start + 1)) "$file" | head -c $range_size | \
     $rg_cmd -e "$query_regex"
 else
   # Search entire file

data/lib/onlylogs/version.rb

@@ -1,3 +1,3 @@
 module Onlylogs
-  VERSION = "0.7.0"
+  VERSION = "0.8.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: onlylogs
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - Alessandro Rodi