onlylogs 0.10.0 → 0.12.0

data/app/views/onlylogs/shared/_shortcuts_dialog.html.erb

@@ -0,0 +1,62 @@
+<%# locals: (modifier_label:) %>
+<%
+  sections = [
+    ["Hold #{modifier_label} and press – anywhere, also while typing", [
+      [%w[L], "Live"],
+      [%w[S], "Search"],
+      [%w[Q], "Go to the query"],
+      [%w[X], "Clear the query"],
+      [%w[V], "Saved searches: apply, rename or delete one"],
+      [%w[F], "Save the query (the star)"],
+      [%w[R], "Regexp on/off"],
+      [%w[I], "Ignore case on/off"],
+      [%w[A], "Autoscroll on/off (live)"],
+      [%w[W], "Search the whole file (live)"],
+      [%w[G], "Range slider (search), then ← →"],
+      [%w[0], "Reset the range (search)"],
+      [%w[.], "Stop the search"],
+      [%w[C], "Clear the lines"],
+      [%w[J], "Jump into the lines"]
+    ]],
+    ["When not typing", [
+      [%w[/], "Go to the query"],
+      [%w[↓ j], "Jump into the lines"],
+      [%w[?], "This help"]
+    ]],
+    ["In the query", [
+      [%w[Enter Esc], "Jump into the lines"]
+    ]],
+    ["In the lines", [
+      [%w[↑ ↓ k j], "Previous / next line"],
+      [%w[PgUp PgDn], "A page up / down"],
+      [%w[Home End g G], "First / last line"],
+      [%w[← →], "Select the previous / next word"],
+      [["Shift ← →"], "Grow the selection by a word"],
+      [%w[Enter], "Search the selection, or show around this line"],
+      [%w[y], "Copy the line"],
+      [%w[Esc], "Clear the selection, then leave the lines"]
+    ]]
+  ]
+%>
+<dialog class="onlylogs-shortcuts-dialog" data-keyboard-shortcuts-target="dialog" closedby="any"
+        aria-labelledby="onlylogs-shortcuts-title">
+  <form method="dialog" class="onlylogs-shortcuts-dialog__header">
+    <h2 id="onlylogs-shortcuts-title">Keyboard shortcuts</h2>
+    <button aria-label="Close" autofocus>×</button>
+  </form>
+  <div class="onlylogs-shortcuts-dialog__sections">
+    <% sections.each do |title, shortcuts| %>
+      <section>
+        <h3><%= title %></h3>
+        <dl>
+          <% shortcuts.each do |keys, description| %>
+            <div>
+              <dt><% keys.each do |key| %><kbd><%= key %></kbd><% end %></dt>
+              <dd><%= description %></dd>
+            </div>
+          <% end %>
+        </dl>
+      </section>
+    <% end %>
+  </div>
+</dialog>

data/bin/super_grep

@@ -20,6 +20,10 @@ while [[ $# -gt 0 ]]; do
       end_position="$2"
       shift 2
       ;;
+    --case-insensitive)
+      case_insensitive=true
+      shift
+      ;;
     --regexp)
       regexp_mode=true
       shift
@@ -60,6 +64,9 @@ grep_cmd=(grep -E -h)             # -h = no filename (portable)
 if [ -n "$max_matches" ]; then
   grep_cmd+=(-m "$max_matches")   # -m = max count (portable)
 fi
+if [ "$case_insensitive" = true ]; then
+  grep_cmd+=(-i)
+fi
 
 # Handle byte range if specified
 if [ -n "$start_position" ] || [ -n "$end_position" ]; then

data/bin/super_ripgrep

@@ -20,6 +20,10 @@ while [[ $# -gt 0 ]]; do
       end_position="$2"
       shift 2
       ;;
+    --case-insensitive)
+      case_insensitive=true
+      shift
+      ;;
     --regexp)
       regexp_mode=true
       shift
@@ -62,6 +66,7 @@ query_regex="${query_regex//$placeholder/$actual_color_regex}"
 # --byte-offset: include byte offset in output for the logs-around-line feature
 rg_cmd="rg --color=never --no-filename --byte-offset --no-mmap"
 [ -n "$max_matches" ] && rg_cmd="$rg_cmd --max-count=$max_matches"
+[ "$case_insensitive" = true ] && rg_cmd="$rg_cmd --ignore-case"
 
 # Handle byte range if specified
 if [ -n "$start_position" ] || [ -n "$end_position" ]; then

data/config/routes.rb

@@ -2,4 +2,5 @@ Onlylogs::Engine.routes.draw do
   root "logs#index"
   resources :logs, only: [:index]
   get "download", to: "logs#download", as: :download_log
+  resources :queries, only: [:index, :create, :show, :update, :destroy]
 end

data/lib/onlylogs/configuration.rb

@@ -4,7 +4,7 @@ module Onlylogs
   class Configuration
     attr_accessor :log_file_patterns, :default_log_file_path, :basic_auth_user, :basic_auth_password,
       :parent_controller, :disable_basic_authentication, :ripgrep_enabled, :editor,
-      :max_line_matches, :search_timeout
+      :max_line_matches, :search_timeout, :database_dir, :viewer_parent_controller, :authorize_log_file
 
     # Seconds a viewer search may run before it is stopped. Bounded by default:
     # a search holds a CPU for as long as it runs, and an unbounded one over a
@@ -19,11 +19,14 @@ module Onlylogs
       @basic_auth_user = default_basic_auth_user
       @basic_auth_password = default_basic_auth_password
       @parent_controller = nil
+      @viewer_parent_controller = nil
+      @authorize_log_file = nil
       @disable_basic_authentication = false
       @ripgrep_enabled = default_ripgrep_enabled
       @editor = nil
       @max_line_matches = 100000
       @search_timeout = DEFAULT_SEARCH_TIMEOUT
+      @database_dir = ".onlylogs"
     end
 
     def configure
@@ -129,6 +132,16 @@ module Onlylogs
     configuration.parent_controller
   end
 
+  def self.viewer_parent_controller
+    configuration.viewer_parent_controller
+  end
+
+  # Runs in the controller, so it can reach the host's current user.
+  def self.log_file_authorized?(controller, file_path)
+    hook = configuration.authorize_log_file
+    hook.nil? || controller.instance_exec(file_path, &hook).present?
+  end
+
   def self.disable_basic_authentication?
     configuration.disable_basic_authentication
   end

data/lib/onlylogs/http_device.rb

@@ -7,15 +7,17 @@ require_relative "spool"
 # A Logger log device that sends log lines to onlylogs.io (or any Vector-compatible sink) directly
 # via HTTP.
 #
+# Delivery is fire and forget: a batch the drain does not accept within the timeouts is lost.
 # When the drain is unreachable or unresponsive, we do two things to protect the app:
 # * an upper bound to the in-memory queue: log lines can never accumulate without limit and exhaust memory
 # * cooldown: once the drain is known to be failing we stop attempting
 #   requests for a cooldown period instead of blocking on every send for the full
 #   read timeout (a down host accepts the TCP/TLS connection but never answers).
 #
-# By default an on-disk Spool buffers any batch we could not deliver and replays it once the
-# drain recovers, so a transient outage or a restart does not lose logs. It is on by default
-# (set ONLYLOGS_SPOOL_DIR empty to disable) and bounded by bytes; see Onlylogs::Spool.
+# ONLYLOGS_SPOOL_ENABLED=true (also 1, yes or on, in any case) opts into an on-disk Spool that
+# buffers any batch we could not deliver and replays it once the drain recovers, so a transient
+# outage or a restart does not lose logs. It lives in ONLYLOGS_SPOOL_DIR and is bounded by bytes;
+# see Onlylogs::Spool.
 #
 # Every write checks that a sender thread is alive in the current process and starts one if not:
 # * The device is usually built in the Puma master (production.rb runs before the workers are
@@ -28,11 +30,12 @@ require_relative "spool"
 #
 # Not every failure is worth retrying. The drain's answer decides what happens to a batch:
 # * 2xx: delivered.
-# * 429: the drain is overloaded and asks us to slow down. Pause for Retry-After (or a cooldown)
-#   and keep the batch on disk; nothing is lost, it is just late.
+# * 429: the drain is overloaded and asks us to slow down. Pause for Retry-After (or a cooldown);
+#   with a spool the batch is kept on disk and is just late, without one it is lost.
 # * other 4xx: the drain will never accept this batch (unknown token, paused project, body too
 #   big). Retrying cannot help, so the batch is dropped and a warning says why.
-# * 5xx, timeouts, connection errors: retryable. Count towards opening the circuit and spool.
+# * 5xx, timeouts, connection errors: an outage. Count towards opening the circuit; the batch
+#   goes to the spool if there is one.
 module Onlylogs
   class HttpDevice
     # The drain answered with a 4xx other than 429: the batch itself is the problem, not the drain.
@@ -85,6 +88,7 @@ module Onlylogs
       read_timeout: ENV.fetch("ONLYLOGS_READ_TIMEOUT", DEFAULT_READ_TIMEOUT),
       circuit_cooldown: ENV.fetch("ONLYLOGS_CIRCUIT_COOLDOWN", CIRCUIT_COOLDOWN),
       keep_alive_timeout: ENV.fetch("ONLYLOGS_KEEP_ALIVE_TIMEOUT", DEFAULT_KEEP_ALIVE_TIMEOUT),
+      spool_enabled: ENV.fetch("ONLYLOGS_SPOOL_ENABLED", false),
       spool_dir: ENV.fetch("ONLYLOGS_SPOOL_DIR", default_spool_dir),
       spool_max_bytes: ENV.fetch("ONLYLOGS_SPOOL_MAX_BYTES", Spool::DEFAULT_MAX_BYTES)
     )
@@ -99,6 +103,7 @@ module Onlylogs
       @read_timeout = float_setting("ONLYLOGS_READ_TIMEOUT", read_timeout, DEFAULT_READ_TIMEOUT)
       @circuit_cooldown = float_setting("ONLYLOGS_CIRCUIT_COOLDOWN", circuit_cooldown, CIRCUIT_COOLDOWN)
       @keep_alive_timeout = float_setting("ONLYLOGS_KEEP_ALIVE_TIMEOUT", keep_alive_timeout, DEFAULT_KEEP_ALIVE_TIMEOUT)
+      @spool_enabled = boolean_setting("ONLYLOGS_SPOOL_ENABLED", spool_enabled, false)
       @spool_dir = spool_dir
       @spool_max_bytes = integer_setting("ONLYLOGS_SPOOL_MAX_BYTES", spool_max_bytes, Spool::DEFAULT_MAX_BYTES)
       @supervisor_mutex = Mutex.new
@@ -159,6 +164,17 @@ module Onlylogs
       value.nil? || value.to_s.strip.empty?
     end
 
+    TRUTHY = %w[true 1 yes on].freeze
+    FALSY = %w[false 0 no off].freeze
+
+    def boolean_setting(name, value, default)
+      text = value.to_s.strip.downcase
+      return true if TRUTHY.include?(text)
+      return false if text.empty? || FALSY.include?(text)
+
+      fallback_setting(name, value, default, "true or false")
+    end
+
     def integer_setting(name, value, default, min: 1)
       number = Integer(value, exception: false)
       return number if number && number >= min
@@ -230,7 +246,7 @@ module Onlylogs
       @http_mutex = Mutex.new
       @http = nil
       @sender_thread = nil
-      @spool = build_spool(@spool_dir, @spool_max_bytes) if @drain_url
+      @spool = build_spool(@spool_dir, @spool_max_bytes) if @drain_url && @spool_enabled
       @consecutive_failures = 0
       @circuit_open_until = nil
       @dropped = 0
@@ -326,8 +342,7 @@ module Onlylogs
       body = lines.join("\n")
 
       # Drain is known to be down: skip the request entirely so we don't block for the full read
-      # timeout on every batch. Buffer the batch so the cooldown does not cost us data (without a
-      # spool configured, spool_write is a no-op and the batch is dropped — best-effort logging).
+      # timeout on every batch. The batch is dropped, unless a spool is configured to keep it.
       if circuit_open?
         spool_write(body)
         return
@@ -373,8 +388,9 @@ module Onlylogs
       end
     end
 
+    # A spool that cannot be set up must not stop the app from logging: it is disabled with a warning.
     def build_spool(dir, max_bytes)
-      return if dir.nil? || dir.to_s.strip.empty?
+      raise ArgumentError, "ONLYLOGS_SPOOL_DIR is empty" if blank?(dir)
 
       Spool.new(dir: dir, max_bytes: max_bytes)
     rescue => e
@@ -382,9 +398,8 @@ module Onlylogs
       nil
     end
 
-    # The spool is on by default. It lives under the app's tmp dir, which survives a drain outage
-    # while the app keeps running; point ONLYLOGS_SPOOL_DIR at a persistent volume to also survive
-    # redeploys, or set it empty to disable.
+    # Under the app's tmp dir, which survives a drain outage while the app keeps running; point
+    # ONLYLOGS_SPOOL_DIR at a persistent volume to also survive redeploys.
     def default_spool_dir
       base = if defined?(Rails) && Rails.respond_to?(:root) && Rails.root
         Rails.root.to_s

data/lib/onlylogs/version.rb

@@ -1,3 +1,3 @@
 module Onlylogs
-  VERSION = "0.10.0"
+  VERSION = "0.12.0"
 end

data/lib/onlylogs.rb

@@ -1,3 +1,4 @@
+require "sqlite_shards"
 require "onlylogs/version"
 require "onlylogs/configuration"
 require "onlylogs/engine"
@@ -18,6 +19,9 @@ require "onlylogs/http_logger"
 # loader.setup
 
 module Onlylogs
+  # Not autoloaded: referenced from class bodies (rescue_from, subclassing).
+  class Error < StandardError; end
+
   class ForbiddenPathError < StandardError; end
 
   if defined?(Importmap)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: onlylogs
 version: !ruby/object:Gem::Version
-  version: 0.10.0
+  version: 0.12.0
 platform: ruby
 authors:
 - Alessandro Rodi
@@ -23,6 +23,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '8.0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: sqlite_shards
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
 description: This gem includes all the tools needed to view and stream you log files
   directly on a web interface.
 email:
@@ -49,15 +77,21 @@ files:
 - app/assets/stylesheets/onlylogs/clusterize.css
 - app/channels/onlylogs/application_cable/channel.rb
 - app/channels/onlylogs/logs_channel.rb
+- app/controllers/concerns/onlylogs/authentication.rb
 - app/controllers/onlylogs/application_controller.rb
 - app/controllers/onlylogs/logs_controller.rb
+- app/controllers/onlylogs/queries_controller.rb
+- app/controllers/onlylogs/viewer_controller.rb
 - app/helpers/onlylogs/application_helper.rb
 - app/javascript/onlylogs/application.js
 - app/javascript/onlylogs/controllers/application.js
 - app/javascript/onlylogs/controllers/auto_submit_controller.js
 - app/javascript/onlylogs/controllers/index.js
 - app/javascript/onlylogs/controllers/keyboard_shortcuts_controller.js
+- app/javascript/onlylogs/controllers/log_cursor_controller.js
 - app/javascript/onlylogs/controllers/log_streamer_controller.js
+- app/javascript/onlylogs/controllers/match_highlight_controller.js
+- app/javascript/onlylogs/controllers/queries_controller.js
 - app/javascript/onlylogs/controllers/range_slider_controller.js
 - app/javascript/onlylogs/controllers/text_selection_controller.js
 - app/jobs/onlylogs/application_job.rb
@@ -66,6 +100,8 @@ files:
 - app/models/onlylogs/file.rb
 - app/models/onlylogs/file_path_parser.rb
 - app/models/onlylogs/grep.rb
+- app/models/onlylogs/queries_record.rb
+- app/models/onlylogs/query.rb
 - app/models/onlylogs/secure_file_path.rb
 - app/views/home/show.html.erb
 - app/views/layouts/onlylogs/application.html.erb
@@ -73,6 +109,7 @@ files:
 - app/views/onlylogs/shared/_log_container.html.erb
 - app/views/onlylogs/shared/_log_container_styles.html.erb
 - app/views/onlylogs/shared/_range_slider.html.erb
+- app/views/onlylogs/shared/_shortcuts_dialog.html.erb
 - bin/onlylogs_sidecar
 - bin/super_grep
 - bin/super_ripgrep