pg_reports 0.9.0 → 1.0.0

data/lib/pg_reports/query_monitor.rb

@@ -3,111 +3,155 @@
 require "singleton"
 require "securerandom"
 require "json"
+require "time"
+require "active_support/cache"
 
 module PgReports
+  # Captures the SQL the host application runs, across all of its processes.
+  #
+  # Web workers, job processes and the dashboard's own request can each be a
+  # different process, so nothing the dashboard depends on may live in this
+  # object's memory: the session and the captured queries live in a shared cache
+  # (see #store), and every process keeps a subscriber that only does real work
+  # while that cache says a session is running.
+  #
+  # The captured queries are a ring: an entry's key is its sequence number modulo
+  # query_monitor_max_queries, so a session's footprint in the cache is bounded
+  # no matter how much SQL the application runs.
+  #
+  # Request threads only append to an in-memory batch; a short-lived background
+  # thread moves batches to the cache, so capturing adds no cache round trip to
+  # the host application's queries.
+  #
+  # A session has one of two sources. :notifications is the above. :activity has
+  # the background thread of the process that started the session sample
+  # pg_stat_activity on the selected database, feeding the same ring; it is the
+  # only source standalone mode has, because there is no application to subscribe
+  # to. A backend only remembers its latest statement, so a sample sees every
+  # client of the database but misses statements replaced between two samples.
   class QueryMonitor
     include Singleton
 
-    CACHE_KEY_ENABLED = "pg_reports:query_monitor:enabled"
-    CACHE_KEY_SESSION_ID = "pg_reports:query_monitor:session_id"
-    CACHE_TTL = 24.hours
+    KEY_PREFIX = "pg_reports:query_monitor"
+    CACHE_KEY_STATE = "#{KEY_PREFIX}:state"
+    CACHE_KEY_LAST_SESSION = "#{KEY_PREFIX}:last_session"
 
-    def initialize
-      @subscriber = nil
-      @mutex = Mutex.new
-      @queries = []
-      @handling_event = false
+    # How long captured queries stay readable (and downloadable) after a session ends
+    BUFFER_RETENTION = 3600
+    FLUSH_INTERVAL = 0.5 # seconds between moves of a batch to the cache
+    STATE_SYNC_INTERVAL = 1.0 # seconds between checks for a session started elsewhere
+    MAX_SQL_LENGTH = 10_000 # bounds an entry's size: a bulk INSERT can be megabytes
+    LOG_PERMISSIONS = 0o600 # captured SQL is for whoever runs the application
+
+    # Standalone sets this on its connections, which is how samples leave them out
+    APPLICATION_NAME = "pg_reports"
+    INSUFFICIENT_PRIVILEGE = "<insufficient privilege>"
+    # What clients run around their queries; ";" is how the driver checks a connection
+    CONNECTION_NOISE = /\A\s*(;|(BEGIN|COMMIT|ROLLBACK|START TRANSACTION)\b)/i
 
-      # Local state — used by the event handler to avoid cache reads
-      # (which generate SQL events and cause infinite recursion with DB-backed caches)
-      @enabled = false
-      @session_id = nil
+    # Set while this thread is doing the monitor's own cache work. With a
+    # database-backed cache (SolidCache) that work is itself SQL, which must not
+    # be captured, or each capture would trigger the next one.
+    INTERNAL_FLAG = :pg_reports_query_monitor_internal
 
-      sync_from_cache
-      ensure_subscription_if_enabled
+    def initialize
+      @subscriber = nil
+      @lock = Mutex.new
+      @pending = []
+      @dropped = 0
+      @active_session = nil
+      @next_sync_at = 0.0
+      @flusher = nil
+      @sample_context = nil
+      @sampling_session = nil
+      @sampled_queries = nil
+      @hidden_sessions = 0
+      @capture_error = nil
     end
 
-    attr_reader :enabled
+    # Idempotent. Called once per process at boot, so that a process which never
+    # serves a dashboard request still captures once a session starts.
+    def subscribe
+      @lock.synchronize do
+        @subscriber ||= ActiveSupport::Notifications.subscribe("sql.active_record") do |name, started, finished, unique_id, payload|
+          handle_sql_event(name, started, finished, unique_id, payload)
+        end
+      end
+    end
 
-    attr_reader :session_id
+    def enabled
+      !read_state.nil?
+    end
 
-    def start
-      @mutex.synchronize do
-        if enabled
-          Rails.logger.info("PgReports: Monitoring already active, session_id=#{session_id}") if defined?(Rails)
-          return {success: false, message: "Monitoring already active"}
-        end
+    def session_id
+      read_state&.fetch(:session_id, nil)
+    end
 
-        new_session_id = SecureRandom.uuid
-        @queries = []
+    def start(source: nil)
+      source = (source.presence || available_sources.first).to_s
+      return {success: false, error: "Unknown monitoring source: #{source}"} unless available_sources.map(&:to_s).include?(source)
 
-        # Update local state first (used by event handler — no cache round-trip)
-        @enabled = true
-        @session_id = new_session_id
+      subscribe if source == "notifications"
 
-        # Store state in cache so other processes can see it
-        cache_write(CACHE_KEY_ENABLED, true)
-        cache_write(CACHE_KEY_SESSION_ID, new_session_id)
+      return {success: false, message: "Monitoring already active"} if read_state
 
-        Rails.logger.info("PgReports: Monitoring started, session_id=#{new_session_id}") if defined?(Rails)
+      new_session_id = SecureRandom.uuid
+      claim_session(new_session_id, source)
 
-        # Subscribe to sql.active_record events in THIS process
-        ensure_subscription
+      # unless_exist lets concurrent starts race safely; the read decides who won
+      # and also tells us whether the cache accepted the write at all.
+      winner = read_state
+      return {success: false, error: "Query monitor could not write to the cache store"} unless winner
+      return {success: false, message: "Monitoring already active"} unless winner[:session_id] == new_session_id
 
-        # Write session start marker to file
-        write_session_marker("session_start")
+      # Better to refuse now than to run a session nothing can be read back from
+      sequence(new_session_id)
+      verify_activity_tracking if source == "activity"
 
-        {success: true, message: "Query monitoring started", session_id: new_session_id}
+      if source == "activity"
+        prepare_sampling(new_session_id)
+      else
+        @active_session = new_session_id
       end
-    rescue => e
-      @enabled = false
-      @session_id = nil
-      cache_write(CACHE_KEY_ENABLED, false)
-      {success: false, error: e.message}
+      ensure_flusher
+      write_session_marker("session_start", new_session_id)
+
+      {success: true, message: "Query monitoring started", session_id: new_session_id}
+    rescue StandardError, NotImplementedError => e
+      internal { store.delete(CACHE_KEY_STATE) } if new_session_id
+      {success: false, error: "Query monitor cannot use this cache store: #{e.message}"}
     end
 
     def stop
-      @mutex.synchronize do
-        unless enabled
-          return {success: false, message: "Monitoring not active"}
-        end
-
-        current_session_id = @session_id
+      state = read_state
+      return {success: false, message: "Monitoring not active"} unless state
 
-        # Clear local state immediately — stops event handler from processing
-        @enabled = false
-        @session_id = nil
-
-        # Unsubscribe from notifications in THIS process
-        if @subscriber
-          ActiveSupport::Notifications.unsubscribe(@subscriber)
-          @subscriber = nil
-        end
+      internal { store.delete(CACHE_KEY_STATE) }
+      @active_session = nil
+      @sampling_session = nil
+      flush_pending
+      write_session_marker("session_end", state[:session_id])
 
-        # Write session end marker to file
-        write_session_marker("session_end", current_session_id)
-
-        # Flush queries to file
-        flush_to_file
-
-        # Clear state from cache so other processes see it
-        cache_delete(CACHE_KEY_ENABLED)
-        cache_delete(CACHE_KEY_SESSION_ID)
-
-        @queries = []
-
-        {success: true, message: "Query monitoring stopped", session_id: current_session_id}
-      end
+      {success: true, message: "Query monitoring stopped", session_id: state[:session_id]}
     rescue => e
       {success: false, error: e.message}
     end
 
     def status
+      state = read_state
+
       {
-        enabled: enabled,
-        session_id: session_id,
-        query_count: @queries.size,
-        history_available: history_available?
+        enabled: !state.nil?,
+        session_id: state && state[:session_id],
+        query_count: total_queries,
+        history_available: history_available?,
+        expires_in: state && [state[:expires_at] - Time.now.to_i, 0].max,
+        scope: capture_scope,
+        source: state ? state[:source] : available_sources.first.to_s,
+        available_sources: available_sources,
+        hidden_sessions: @hidden_sessions,
+        capture_error: @capture_error,
+        max_queries: ring_size
       }
     end
 
@@ -118,20 +162,27 @@ module PgReports
       log_file_enabled? && File.exist?(log_file_path)
     end
 
-    def queries(limit: nil, session_id: nil)
-      result = @queries.dup
+    # Captured queries, oldest first, for the running session or else the most
+    # recent one. `since` is a sequence number: only later entries are returned.
+    def queries(limit: nil, session_id: nil, since: 0)
+      flush_pending
 
-      # Filter by session_id if provided
-      if session_id
-        result = result.select { |q| q[:session_id] == session_id }
-      end
+      session = session_id.presence || current_or_last_session
+      return [] unless session
 
-      # Limit results if requested
-      if limit
-        result = result.last(limit)
-      end
+      newest = sequence(session)
+      oldest = [newest - ring_size + 1, since.to_i + 1, 1].max
+      oldest = [oldest, newest - limit + 1].max if limit
+      return [] if newest < oldest
+
+      keys = (oldest..newest).to_h { |seq| [seq, entry_key(session, seq)] }
+      found = internal { store.read_multi(*keys.values) }
 
-      result
+      # A slot can already hold a newer lap of the ring, or have been evicted
+      keys.filter_map do |seq, key|
+        entry = found[key]
+        entry if entry && entry[:seq] == seq
+      end
     end
 
     def load_from_log(session_id: nil, limit: nil)
@@ -150,122 +201,352 @@ module PgReports
             next if session_id && entry[:session_id] != session_id
 
             queries << entry
+            queries.shift if limit && queries.size > limit
           rescue JSON::ParserError
             # Skip malformed lines
             next
           end
         end
 
-        # Limit results if requested
-        queries = queries.last(limit) if limit
-
         queries
       rescue => e
-        Rails.logger.warn("PgReports: Failed to load queries from log: #{e.message}") if defined?(Rails)
+        log_warning("Failed to load queries from log: #{e.message}")
         []
       end
     end
 
     private
 
-    # Cache helpers - work with or without Rails.cache
-    def cache_read(key)
-      return nil unless cache_available?
-      Rails.cache.read(key)
+    # The cache every process must share for the monitor to see all of them.
+    # Without one (Rails' NullStore default, or the per-process MemoryStore) this
+    # falls back to a private store, so one process still works and #status can
+    # report that only one process is covered.
+    def store
+      # A session belongs to the standalone process whose thread is sampling for
+      # it, so it must not outlive that process in a cache on disk. This cannot
+      # depend on the session's source: the store is how the source is learned.
+      return local_store if PgReports.config.standalone
+
+      PgReports.config.query_monitor_cache_store ||
+        (shared_store? ? Rails.cache : local_store)
+    end
+
+    def shared_store?
+      return true if PgReports.config.query_monitor_cache_store
+
+      cache = Rails.cache if defined?(Rails) && Rails.respond_to?(:cache)
+      return false if cache.nil?
+
+      !cache.is_a?(ActiveSupport::Cache::NullStore) && !cache.is_a?(ActiveSupport::Cache::MemoryStore)
+    end
+
+    def local_store
+      @local_store ||= ActiveSupport::Cache::MemoryStore.new
+    end
+
+    def available_sources
+      PgReports.config.query_monitor_available_sources
+    end
+
+    def capture_scope
+      return "database" if PgReports.config.standalone
+
+      shared_store? ? "shared" : "process"
+    end
+
+    def internal
+      previous = Thread.current[INTERNAL_FLAG]
+      Thread.current[INTERNAL_FLAG] = true
+      yield
+    ensure
+      Thread.current[INTERNAL_FLAG] = previous
+    end
+
+    def read_state
+      internal { store.read(CACHE_KEY_STATE) }
     rescue => e
-      Rails.logger.warn("PgReports: Cache read failed: #{e.message}") if defined?(Rails.logger)
+      log_warning("Failed to read monitoring state: #{e.message}")
       nil
     end
 
-    def cache_write(key, value)
-      return false unless cache_available?
-      Rails.cache.write(key, value, expires_in: CACHE_TTL)
+    def claim_session(new_session_id, source)
+      duration = PgReports.config.query_monitor_max_duration.to_i
+      state = {session_id: new_session_id, source: source, expires_at: Time.now.to_i + duration}
+
+      internal do
+        # Only the raw form of a counter can be incremented on Redis and Memcached
+        store.write(sequence_key(new_session_id), 0, raw: true, expires_in: retention)
+        store.write(CACHE_KEY_STATE, state, unless_exist: true, expires_in: duration)
+        store.write(CACHE_KEY_LAST_SESSION, new_session_id, expires_in: retention) if read_state&.fetch(:session_id) == new_session_id
+      end
+    end
+
+    def current_or_last_session
+      session_id || internal { store.read(CACHE_KEY_LAST_SESSION) }
+    end
+
+    def total_queries
+      session = current_or_last_session
+      session ? sequence(session) : 0
+    end
+
+    # Asked of the cache through increment, not read: the counter is stored raw,
+    # and a cache that wraps another (RailsBrotliCache) deserializes what it reads.
+    # The expiry is repeated because some stores rewrite the entry on increment.
+    def sequence(session)
+      internal { store.increment(sequence_key(session), 0, raw: true, expires_in: retention) }.to_i
+    end
+
+    def retention
+      PgReports.config.query_monitor_max_duration.to_i + BUFFER_RETENTION
+    end
+
+    def sequence_key(session)
+      "#{KEY_PREFIX}:#{session}:seq"
+    end
+
+    def entry_key(session, seq)
+      "#{KEY_PREFIX}:#{session}:entry:#{seq % ring_size}"
+    end
+
+    def ring_size
+      [PgReports.config.query_monitor_max_queries.to_i, 1].max
+    end
+
+    def handle_sql_event(name, started, finished, unique_id, payload)
+      return if Thread.current[INTERNAL_FLAG]
+
+      session = capturing_session
+      return unless session
+      return if skip_by_payload?(payload)
+
+      locations = caller_locations(0, 60)
+      return if query_from_pg_reports?(locations)
+
+      add_to_batch(
+        type: "query",
+        session_id: session,
+        duration_ms: ((finished - started) * 1000).round(2),
+        name: payload[:name],
+        timestamp: Time.current.iso8601(3),
+        **statement_fields(payload[:sql].to_s) { extract_source_location(locations) }
+      )
     rescue => e
-      Rails.logger.warn("PgReports: Cache write failed: #{e.message}") if defined?(Rails.logger)
-      false
+      # Runs inside the host application's query; it must never break it
+      log_warning("Failed to capture a query: #{e.message}")
+    end
+
+    # This runs on every query of every process, including when nothing is being
+    # monitored, so the cache is consulted at most once per STATE_SYNC_INTERVAL.
+    def capturing_session
+      now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      if now >= @next_sync_at
+        @next_sync_at = now + STATE_SYNC_INTERVAL
+        sync_session
+      end
+
+      @active_session
+    end
+
+    def sync_session
+      @active_session = notifications_session(read_state)
+      ensure_flusher if @active_session
     end
 
-    def cache_delete(key)
-      return false unless cache_available?
-      Rails.cache.delete(key)
+    # An :activity session is left to the process sampling for it, or events
+    # would record the same application SQL a second time.
+    def notifications_session(state)
+      state[:session_id] if state && state[:source] == "notifications"
+    end
+
+    def add_to_batch(entry)
+      @lock.synchronize do
+        @pending << entry
+        next unless @pending.size > ring_size
+
+        @pending.shift
+        @dropped += 1
+      end
+    end
+
+    # Runs only while a session is active. Forked workers do not inherit the
+    # thread, so alive? is also what restarts it after a fork.
+    def ensure_flusher
+      @lock.synchronize do
+        return if @flusher&.alive?
+
+        @flusher = Thread.new { run_flusher }
+        @flusher.name = "pg_reports-query-monitor"
+        @flusher.report_on_exception = false
+      end
+    end
+
+    def run_flusher
+      Thread.current[INTERNAL_FLAG] = true
+
+      loop do
+        sleep FLUSH_INTERVAL
+        sample_activity if @sampling_session
+        flush_pending
+
+        state = read_state
+        @active_session = notifications_session(state)
+        @sampling_session = nil unless state && state[:session_id] == @sampling_session
+        break unless @active_session || @sampling_session
+      end
+
+      flush_pending
     rescue => e
-      Rails.logger.warn("PgReports: Cache delete failed: #{e.message}") if defined?(Rails.logger)
-      false
+      log_warning("Query monitor flusher stopped: #{e.message}")
     end
 
-    def cache_available?
-      defined?(Rails) && defined?(Rails.cache)
+    def flush_pending
+      batch, dropped = @lock.synchronize do
+        taken = [@pending, @dropped]
+        @pending = []
+        @dropped = 0
+        taken
+      end
+      return if batch.empty?
+
+      batch.group_by { |entry| entry[:session_id] }.each do |session, entries|
+        write_entries(session, entries, dropped)
+        dropped = 0
+      end
+      append_to_log(batch)
+      @capture_error = nil
+    rescue StandardError, NotImplementedError => e
+      report_capture_error("Failed to store captured queries: #{e.message}")
     end
 
-    # Sync local state from shared cache (called on initialize for multi-process support)
-    def sync_from_cache
-      @enabled = enabled
-      @session_id = session_id
+    # Reserving the numbers with one atomic increment is what lets any number of
+    # processes write into the same ring.
+    # Queries dropped in a burst still take a number, so the total stays honest.
+    def write_entries(session, entries, dropped = 0)
+      internal do
+        newest = store.increment(sequence_key(session), entries.size + dropped, raw: true, expires_in: retention)
+        next unless newest
+
+        first = newest - entries.size + 1
+        numbered = entries.each_with_index.to_h do |entry, index|
+          seq = first + index
+          [entry_key(session, seq), entry.merge(seq: seq)]
+        end
+        store.write_multi(numbered, expires_in: retention)
+      end
+    end
+
+    # A thread starts outside the request's target and database, so the viewer's
+    # selection has to be carried over explicitly.
+    def prepare_sampling(session)
+      @sample_context = current_context
+      @sampling_session = session
+      @sampled_queries = nil
+      @hidden_sessions = 0
+    end
+
+    def sample_activity
+      rows = read_activity
+      visible, hidden = rows.partition { |row| row["query"] != INSUFFICIENT_PRIVILEGE }
+      @hidden_sessions = hidden.size
+
+      # The first sample only shows what ran before this session began
+      capture_finished_queries(visible) if @sampled_queries
+
+      @sampled_queries = visible.to_h { |row| [row["pid"], row["query_start"]] }
+    rescue => e
+      report_capture_error("Failed to sample pg_stat_activity: #{e.message}")
     end
 
-    # Ensure this process is subscribed to notifications if monitoring is enabled
-    def ensure_subscription_if_enabled
-      return unless @enabled
-      ensure_subscription
+    def report_capture_error(message)
+      @capture_error = message
+      log_warning(message)
     end
 
-    def ensure_subscription
-      return if @subscriber # Already subscribed
+    # pg_stat_activity is part of PostgreSQL and needs no extension. This setting
+    # is the one thing that can leave it empty, which would otherwise look like a
+    # database with nothing running.
+    def verify_activity_tracking
+      rows = activity_executor(current_context) { |executor| executor.execute("SHOW track_activities") }
+      return unless rows.first&.fetch("track_activities", nil) == "off"
 
-      @subscriber = ActiveSupport::Notifications.subscribe("sql.active_record") do |name, started, finished, unique_id, payload|
-        handle_sql_event(name, started, finished, unique_id, payload)
+      raise Error, "track_activities is off, so pg_stat_activity shows no queries"
+    end
+
+    # Goes to the pool directly: PgReports.config.connection would lease a
+    # connection to this thread, which would then keep a pool slot for the whole
+    # session. with_connection hands it back after every sample.
+    def read_activity
+      activity_executor(@sample_context) { |executor| executor.execute_from_file(:queries, :activity_sample) }
+    end
+
+    def activity_executor(context)
+      target = PgReports.connection_registry.fetch(context[:target])
+
+      target.ar_class_for(context[:database]).connection_pool.with_connection do |connection|
+        yield Executor.new(connection: connection)
       end
+    end
 
-      Rails.logger.debug("PgReports: Subscribed to sql.active_record in process #{Process.pid}") if defined?(Rails.logger)
+    def current_context
+      registry = PgReports.connection_registry
+      {target: registry.current_name || registry.default_name, database: registry.current_database}
     end
 
-    def handle_sql_event(name, started, finished, unique_id, payload)
-      # Use local @enabled instead of enabled (which hits cache and may generate SQL,
-      # causing infinite recursion with database-backed cache stores like SolidCache)
-      return unless @enabled
-      return if @handling_event
+    def capture_finished_queries(rows)
+      rows.each do |row|
+        next if row["query_start"].nil? || @sampled_queries[row["pid"]] == row["query_start"]
 
-      @handling_event = true
-      begin
-        # Skip if should be filtered
-        return if should_skip?(payload)
-
-        duration_ms = ((finished - started) * 1000).round(2)
-        sql = payload[:sql]
-        query_name = payload[:name]
-
-        # Extract source location
-        source_location = extract_source_location
-
-        # Build query entry
-        query_entry = {
-          type: "query",
-          session_id: @session_id,
-          sql: sql,
-          duration_ms: duration_ms,
-          name: query_name,
-          source_location: source_location,
-          timestamp: Time.current.iso8601
-        }
-
-        add_to_buffer(query_entry)
-      ensure
-        @handling_event = false
+        sql = row["query"].to_s
+        next if sql.match?(CONNECTION_NOISE) || skip_by_payload?(sql: sql, name: row["application_name"])
+
+        add_to_batch(sampled_entry(row, sql))
       end
     end
 
-    def should_skip?(payload)
+    def sampled_entry(row, sql)
+      {
+        type: "query",
+        session_id: @sampling_session,
+        duration_ms: row["duration_ms"].to_f,
+        name: row["application_name"].presence,
+        timestamp: sampled_time(row["query_start"]).iso8601(3),
+        **statement_fields(sql)
+      }
+    end
+
+    # An annotation (Marginalia, Rails QueryLogs) was written by the application
+    # at the call site, so it beats anything worked out from a backtrace. It is
+    # then only noise in the statement itself.
+    def statement_fields(sql)
+      annotation = AnnotationParser.parse(sql)
+      location = {file: annotation[:file], line: annotation[:line].to_i} if annotation[:file]
+
+      {
+        sql: stored_sql(AnnotationParser.strip_annotations(sql)[0, MAX_SQL_LENGTH]),
+        source: AnnotationParser.format_for_display(annotation),
+        source_location: location || (yield if block_given?)
+      }
+    end
+
+    # After the length limit, which keeps the work bounded on this path through the
+    # host application's queries. The redactor treats the cut as the end of a string.
+    def stored_sql(sql)
+      PgReports.config.query_monitor_redact_literals ? SqlRedactor.redact(sql) : sql
+    end
+
+    # A Time or a String, depending on the Rails version
+    def sampled_time(value)
+      value.respond_to?(:iso8601) ? value : Time.parse(value)
+    end
+
+    def skip_by_payload?(payload)
       sql = payload[:sql]
       name = payload[:name]
 
       # Skip if query is from pg_reports itself (check by name)
       return true if name&.start_with?("PgReports")
 
-      # Skip if query is from pg_reports gem (check backtrace)
-      if query_from_pg_reports?
-        return true
-      end
-
       # Skip SCHEMA queries
       return true if name&.start_with?("SCHEMA")
 
@@ -284,9 +565,8 @@ module PgReports
       false
     end
 
-    def query_from_pg_reports?
-      # Check if query originates from pg_reports gem internal code
-      locations = caller_locations(0, 30)
+    # Check if query originates from pg_reports gem internal code
+    def query_from_pg_reports?(locations)
       return false unless locations
 
       locations.any? do |location|
@@ -311,24 +591,21 @@ module PgReports
       end
     end
 
-    def extract_source_location
-      # Get caller locations, skip first few frames (this file, active_support)
-      # Increase limit to 50 to capture more of the stack
-      locations = caller_locations(5, 50)
-
+    def extract_source_location(locations)
       return nil unless locations
 
+      keep_frame = PgReports.config.query_monitor_backtrace_filter
+
       # Find first application code location
-      # Look for paths that are NOT from gems/ruby/railties
       app_location = locations.find do |location|
         path = location.path
 
-        # Skip framework and gem paths
-        next if path.match?(%r{/(gems|ruby|railties)/})
+        next false if path.include?("/query_monitor.rb") || path.start_with?("<internal:")
+        next false unless keep_frame.call(location)
 
         # Skip pg_reports internal paths
-        next if path.match?(%r{/pg_reports/lib/pg_reports/})
-        next if path.match?(%r{/pg_reports/app/controllers/pg_reports/})
+        next false if path.match?(%r{/pg_reports/lib/pg_reports/})
+        next false if path.match?(%r{/pg_reports/app/controllers/pg_reports/})
 
         # This is likely application code
         true
@@ -346,21 +623,11 @@ module PgReports
       nil
     end
 
-    def add_to_buffer(query_entry)
-      @queries << query_entry
-
-      # Trim to max_queries to prevent memory bloat
-      max_queries = PgReports.config.query_monitor_max_queries
-      if @queries.size > max_queries
-        @queries = @queries.last(max_queries)
-      end
-    end
-
     # JSON.generate, not #to_json: ActiveSupport overrides #to_json and (up to
     # Rails 7.0) passes json's removed `quirks_mode` option, so on json >= 3 it
     # raises ArgumentError. The rescue below would swallow that and leave the
     # log silently empty. json's own generator takes no such option.
-    def write_session_marker(marker_type, sid = @session_id)
+    def write_session_marker(marker_type, sid)
       return unless log_file_enabled?
 
       marker = {
@@ -369,25 +636,40 @@ module PgReports
         timestamp: Time.current.iso8601
       }
 
-      File.open(log_file_path, "a") do |f|
-        f.puts JSON.generate(marker)
-      end
+      write_log("#{JSON.generate(marker)}\n")
     rescue => e
       # Silently fail - don't break monitoring if file write fails
-      Rails.logger.warn("PgReports: Failed to write session marker: #{e.message}")
+      log_warning("Failed to write session marker: #{e.message}")
     end
 
-    def flush_to_file
+    def append_to_log(entries)
       return unless log_file_enabled?
-      return if @queries.empty?
 
-      File.open(log_file_path, "a") do |f|
-        @queries.each do |query|
-          f.puts JSON.generate(query)
-        end
-      end
+      write_log(entries.map { |entry| "#{JSON.generate(entry)}\n" }.join)
     rescue => e
-      Rails.logger.warn("PgReports: Failed to flush queries to file: #{e.message}")
+      log_warning("Failed to append queries to file: #{e.message}")
+    end
+
+    # chmod as well as the mode given to open, which only applies to a new file:
+    # a log written by an earlier version stays world-readable otherwise.
+    def write_log(text)
+      rotate_log
+
+      File.open(log_file_path, "a", LOG_PERMISSIONS) do |f|
+        f.write(text)
+        f.chmod(LOG_PERMISSIONS)
+      end
+    end
+
+    # One generation, so the log's footprint is at most twice the limit. Processes
+    # can race here; the loser renames a fresh file over the backup, which costs a
+    # moment of history and nothing else.
+    def rotate_log
+      limit = PgReports.config.query_monitor_log_max_bytes.to_i
+      return unless limit.positive?
+      return unless File.exist?(log_file_path) && File.size(log_file_path) >= limit
+
+      File.rename(log_file_path, "#{log_file_path}.1")
     end
 
     def log_file_enabled?
@@ -397,5 +679,9 @@ module PgReports
     def log_file_path
       PgReports.config.query_monitor_log_file
     end
+
+    def log_warning(message)
+      Rails.logger&.warn("PgReports: #{message}") if defined?(Rails) && Rails.respond_to?(:logger)
+    end
   end
 end