rails-ai-context 5.30.2 → 5.32.0

data/lib/rails_ai_context/tools/query.rb

@@ -198,7 +198,7 @@ module RailsAiContext
         when "csv"
           format_csv(redacted)
         else
-          format_table(redacted)
+          format_table(redacted) + unbounded_note(result)
         end
 
         text_response(output)
@@ -312,16 +312,19 @@ module RailsAiContext
         # Build the combined list ONCE per call and dedupe.
         configured = Array(config.query_redacted_columns).map { |c| c.to_s.downcase }
         suffixed   = SENSITIVE_COLUMN_SUFFIXES.map(&:downcase)
-        # An app whose own column merely looks sensitive (oauth_applications.secret)
-        # exempts it here; the built-in list is frozen and cannot be subtracted from.
-        allowed    = Array(config.query_allowed_columns).map { |c| c.to_s.downcase }
-        ((configured + suffixed).uniq - allowed).each do |col|
+        ((configured + suffixed).uniq - allowed_columns.to_a).each do |col|
           next if col.empty?
           return col if down.match?(/\b#{Regexp.escape(col)}\b/)
         end
         nil
       end
 
+      # An app whose own column merely looks sensitive (oauth_applications.secret)
+      # exempts it by name, from the pre-query check and from result redaction alike.
+      private_class_method def self.allowed_columns
+        Array(config.query_allowed_columns).to_set { |c| c.to_s.downcase }
+      end
+
       # ── Database-level execution (Layer 2) ──────────────────────────
       private_class_method def self.execute_safely(sql, row_limit, timeout_seconds)
         conn = ActiveRecord::Base.connection
@@ -383,29 +386,115 @@ module RailsAiContext
         result
       end
 
+      # sqlite3-ruby cannot interrupt a running statement, so a killable child runs it instead.
       private_class_method def self.execute_sqlite(conn, sql, timeout)
-        raw = conn.raw_connection
+        if (path = sqlite_fork_path(conn))
+          begin
+            return execute_sqlite_in_child(conn, path, sql, timeout)
+          rescue ActiveRecord::StatementInvalid => e
+            raise unless e.message.match?(SQLITE_APP_CONNECTION_ONLY)
+          end
+        end
+
         result = nil
         begin
           conn.execute("PRAGMA query_only = ON")
-          # SQLite has no native statement timeout. Use a progress handler
-          # to abort queries that run too long (checked every 1000 VM steps).
-          deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout
-          if raw.respond_to?(:set_progress_handler)
-            raw.set_progress_handler(1000) do
-              if Process.clock_gettime(Process::CLOCK_MONOTONIC) > deadline
-                1 # non-zero = abort
-              else
-                0
-              end
-            end
-          end
           result = conn.select_all(sql)
         ensure
-          raw.set_progress_handler(0, nil) if raw.respond_to?(:set_progress_handler)
           conn.execute("PRAGMA query_only = OFF")
         end
-        result
+        ResultProxy.new(result.columns, result.rows, true)
+      end
+
+      # Errors from state only the app's own connection has: registered functions,
+      # virtual table modules, collations, an encryption key.
+      SQLITE_APP_CONNECTION_ONLY = /no such (?:function|module|collation)|file is not a database/i
+
+      # A row-capped result is far below this; a larger prefix is a corrupt one.
+      SQLITE_MAX_RESULT_BYTES = 1 << 30
+
+      private_class_method def self.read_before(reader, size, deadline)
+        data = "".b
+        while data.bytesize < size
+          left = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          raise ActiveRecord::StatementInvalid, "SQLite query exceeded the statement timeout" unless left.positive? && IO.select([ reader ], nil, nil, left)
+
+          chunk = reader.read_nonblock([ size - data.bytesize, 1 << 16 ].min, exception: false)
+          raise ActiveRecord::StatementInvalid, "the SQLite query process exited without a result" if chunk.nil?
+
+          data << chunk unless chunk == :wait_readable
+        end
+        data
+      end
+
+      # nil for an in-memory or temporary database, or where there is no fork.
+      private_class_method def self.sqlite_fork_path(conn)
+        return nil unless Process.respond_to?(:fork)
+
+        path = conn.raw_connection.filename("main").to_s
+        path unless path.empty?
+      end
+
+      private_class_method def self.execute_sqlite_in_child(conn, path, sql, timeout)
+        parent = Process.pid
+        extensions = Array(conn.pool.db_config.configuration_hash[:extensions]).map { |ext| (ext.is_a?(String) && ext.safe_constantize) || ext }
+        # sqlite3 2.x warns in every child that inherits a writable handle; Rails 8 silences it the same way.
+        SQLite3::ForkSafety.suppress_warnings! if defined?(SQLite3::ForkSafety)
+
+        deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout
+        reader, writer = IO.pipe.each(&:binmode)
+        pid = fork { answer_in_child(reader, writer, path, sql, timeout, extensions) }
+        writer.close
+        read_child_result(reader, deadline)
+      ensure
+        # Also reached in a child whose fork hooks raised: it must not carry on as a second server.
+        exit!(1) if parent && Process.pid != parent
+        reader&.close
+        writer&.close
+        reap_child(pid) if pid
+      end
+
+      # Writes the length-prefixed result and exits without the parent's at_exit hooks.
+      private_class_method def self.answer_in_child(reader, writer, path, sql, timeout, extensions)
+        reader.close
+        payload = begin
+          db = SQLite3::Database.new(path, readonly: true, extensions: extensions)
+          db.busy_timeout = (timeout * 1000).to_i
+          stmt = db.prepare(sql)
+          [ :ok, stmt.columns, stmt.to_a ]
+        rescue => e
+          [ :error, "#{e.class}: #{e.message}" ]
+        end
+        bytes = Marshal.dump(payload)
+        writer.write([ bytes.bytesize ].pack("Q>"), bytes)
+      ensure
+        exit!(0)
+      end
+
+      # Read by length, not to EOF: any other process forked meanwhile holds a copy of the writer.
+      private_class_method def self.read_child_result(reader, deadline)
+        size = read_before(reader, 8, deadline).unpack1("Q>")
+        status, *rest = begin
+          raise ArgumentError, "result length out of range" if size > SQLITE_MAX_RESULT_BYTES
+
+          Marshal.load(read_before(reader, size, deadline))
+        rescue ArgumentError, TypeError
+          raise ActiveRecord::StatementInvalid, "the SQLite query process exited without a result"
+        end
+        raise ActiveRecord::StatementInvalid, rest.first if status == :error
+
+        ActiveRecord::Result.new(*rest)
+      end
+
+      # KILL: TERM cannot land while `step` holds the GVL, and would run inherited at_exit hooks.
+      # Runs in an ensure, so it rescues everything: nothing here may replace the query's own error.
+      private_class_method def self.reap_child(pid)
+        Process.kill(:KILL, pid) rescue nil
+        Process.wait(pid) rescue nil
+      end
+
+      private_class_method def self.unbounded_note(result)
+        result.is_a?(ResultProxy) && result.unbounded ? UNBOUNDED_SQLITE_NOTE : ""
       end
 
       # ── EXPLAIN execution ────────────────────────────────────────────
@@ -462,7 +551,7 @@ module RailsAiContext
         lines << parsed[:raw]
         lines << "```"
 
-        text_response(lines.join("\n"))
+        text_response(lines.join("\n") + unbounded_note(result))
       rescue ActiveRecord::StatementInvalid => e
         text_response("EXPLAIN failed: #{clean_error_message(e.message)}")
       end
@@ -592,14 +681,15 @@ module RailsAiContext
 
       # ── Column redaction (Layer 4) ──────────────────────────────────
       private_class_method def self.redact_results(result)
-        redacted_cols = config.query_redacted_columns.map(&:downcase).to_set
+        allowed = allowed_columns
+        redacted_cols = config.query_redacted_columns.map(&:downcase).to_set - allowed
+        encrypted_cols = Set.new
 
-        # Auto-redact columns declared with `encrypts` in models
         models_data = cached_context&.dig(:models)
         if models_data.is_a?(Hash)
           models_data.each_value do |data|
             next unless data.is_a?(Hash)
-            (data[:encrypts] || []).each { |col| redacted_cols << col.to_s.downcase }
+            (data[:encrypts] || []).each { |col| encrypted_cols << col.to_s.downcase }
           end
         end
         columns = result.columns
@@ -609,8 +699,9 @@ module RailsAiContext
         sensitive_suffixes = %w[password secret token key digest hash].freeze
         redacted_indices = columns.each_with_index.filter_map { |col, i|
           col_down = col.downcase
-          i if redacted_cols.include?(col_down) ||
-               sensitive_suffixes.any? { |suffix| col_down.end_with?(suffix) || col_down.include?("password") || col_down.include?("secret") || col_down.include?("token") }
+          i if encrypted_cols.include?(col_down) || redacted_cols.include?(col_down) ||
+               (!allowed.include?(col_down) &&
+                sensitive_suffixes.any? { |suffix| col_down.end_with?(suffix) || col_down.include?("password") || col_down.include?("secret") || col_down.include?("token") })
         }
 
         return result if redacted_indices.empty?
@@ -702,8 +793,11 @@ module RailsAiContext
         message.lines.first&.strip || message.strip
       end
 
-      # Quacks like ActiveRecord::Result for redacted output.
-      ResultProxy = Struct.new(:columns, :rows)
+      # Quacks like ActiveRecord::Result for redacted output, or for a SQLite
+      # query that ran in-process with no time limit.
+      ResultProxy = Struct.new(:columns, :rows, :unbounded)
+      UNBOUNDED_SQLITE_NOTE = "\n\n_This SQLite query ran without a time limit: query_timeout needs a file-backed database, " \
+        "a platform with fork, and nothing that only the app's own connection has._"
     end
   end
 end

data/lib/rails_ai_context/tools/read_logs.rb

@@ -22,7 +22,7 @@ module RailsAiContext
           },
           file: {
             type: "string",
-            description: "Log file name (e.g. 'production', 'sidekiq'). Defaults to current Rails.env log. '.log' suffix optional."
+            description: "Log file name (e.g. 'production', 'sidekiq', or a rotated 'development.log.0'). Defaults to current Rails.env log. '.log' suffix optional."
           },
           search: {
             type: "string",
@@ -90,6 +90,10 @@ module RailsAiContext
 
         # Detect format and filter by level
         format = detect_format(raw_lines)
+        if level != "all" && !severity_format?(raw_lines, format)
+          warnings << "most of these lines have no severity field, so they cannot be filtered by level; showing every line"
+          level = "all"
+        end
         filtered = filter_by_level(raw_lines, level, format)
 
         redacted = RailsAiContext::Redaction.redact_log_lines(filtered, search: search)
@@ -119,15 +123,15 @@ module RailsAiContext
 
       # ── Log file resolution ─────────────────────────────────────────
 
+      ROTATED_LOG = /\.log\.\d+\z/
+
       private_class_method def self.resolve_log_file(file_name)
-        name = if file_name
-          File.basename(file_name.to_s.strip.delete("\0").delete_suffix(".log"))
-        else
-          rails_env_name
-        end
+        base = file_name ? File.basename(file_name.to_s.strip.delete("\0")) : rails_env_name.to_s
+        # Logger rotates by size to development.log.0, .1 and so on.
+        base = "#{base.delete_suffix(".log")}.log" unless base.match?(ROTATED_LOG)
 
         # A log is tailed, never read whole, so the per-file cap does not apply.
-        located = RailsAiContext::SafePath.locate(File.join("log", "#{name}.log"), under: rails_app.root.to_s, max_size: Float::INFINITY)
+        located = RailsAiContext::SafePath.locate(File.join("log", base), under: rails_app.root.to_s, max_size: Float::INFINITY)
         located.ok? ? located.realpath : nil
       end
 
@@ -156,18 +160,42 @@ module RailsAiContext
         :standard
       end
 
+      SEVERITY = "DEBUG|INFO|WARN(?:ING)?|ERROR|FATAL|UNKNOWN|ANY"
+      # Only where a formatter writes a severity: Logger::Formatter's "I, [ts]  INFO --",
+      # a leading "INFO"/"[INFO]", "[ts] INFO", "<timestamp> INFO", Sidekiq 6/7's
+      # "pid=.. tid=.. INFO: ", logfmt level=, or semantic_logger's "<timestamp> E [pid:thread]".
+      SEVERITY_FIELD = Regexp.union(
+        /\A\d{4}-\d\d-\d\d[T ][\d:.]+ ([TDIWEF]) \[/,
+        /\A[DIWEFA], \[[^\]]*\]\s+(#{SEVERITY}) -- /o,
+        /\A\[?(#{SEVERITY})\]?(?=[\s:]|\z)/o,
+        /\A\[[^\]]*\]\s+\[?(#{SEVERITY})\]?\s/o,
+        /\A\d{4}-\d\d-\d\d[T ][\d:.,]+(?:Z|[+-]\d\d:?\d\d)?\s+\[?(#{SEVERITY})\]?\s/o,
+        /\A(?:\S+ )?pid=\d+ tid=\S+(?: [\w.]+=\S*)* (#{SEVERITY}): /o,
+        /\b(?:level|severity)=(#{SEVERITY})\b/io
+      )
+
+      # semantic_logger writes the first letter of its level; trace sits below debug.
+      LEVEL_ALIASES = { "WARNING" => "WARN", "T" => "DEBUG", "D" => "DEBUG", "I" => "INFO", "W" => "WARN", "E" => "ERROR", "F" => "FATAL" }.freeze
+      ANSI_COLOR = /\e\[[\d;]*m/
+      BACKTRACE_FRAME = /\S:\d+:in /
+
+      # Filtering by level needs a severity on most lines: one message that
+      # happens to start with "WARN:" in a log that writes no severity must
+      # not turn every other line into its continuation.
+      private_class_method def self.severity_format?(lines, format)
+        heads = lines.reject { |l| l.strip.empty? || l.match?(/\A\s/) || l.match?(BACKTRACE_FRAME) }
+        heads.count { |l| extract_level(l, format) } * 2 > heads.size
+      end
+
       private_class_method def self.extract_level(line, format)
         case format
         when :json
-          match = line.match(/"level"\s*:\s*"(\w+)"/i)
+          match = line.match(/"(?:level|severity|lvl)"\s*:\s*"(\w+)"/i)
           match[1].upcase if match
         when :standard
-          # Rails format: I, [timestamp] INFO -- : message
-          # Or: [2026-03-29 10:00:00] INFO  message
-          match = line.match(/\b(DEBUG|INFO|WARN(?:ING)?|ERROR|FATAL)\b/i)
-          level = match[1].upcase if match
-          level = "WARN" if level == "WARNING"
-          level
+          match = line.gsub(ANSI_COLOR, "").match(SEVERITY_FIELD)
+          level = match&.captures&.compact&.first&.upcase
+          LEVEL_ALIASES.fetch(level, level)
         end
       end
 
@@ -182,7 +210,7 @@ module RailsAiContext
         lines.each do |line|
           level = extract_level(line, format)
           if level
-            rank = LEVEL_HIERARCHY[level] || 0
+            rank = LEVEL_HIERARCHY.fetch(level, LEVEL_HIERARCHY["FATAL"])
             include_continuation = rank >= min_rank
           end
           # Lines without a level are continuations (stack traces)
@@ -197,9 +225,9 @@ module RailsAiContext
       private_class_method def self.available_log_files
         log_dir = File.join(rails_app.root.to_s, "log")
         return [] unless Dir.exist?(log_dir)
-        Dir.glob(File.join(log_dir, "*.log"))
+        Dir.glob(File.join(log_dir, "*.log{,.*}"))
           .map { |f| File.basename(f) }
-          .select { |f| f.match?(/\A[\w.\-]+\.log\z/) } # Only clean filenames (alphanumeric, dots, hyphens, underscores)
+          .select { |f| f.match?(/\A[\w.\-]+\.log(?:\.\d+)?\z/) } # Only clean filenames (alphanumeric, dots, hyphens, underscores)
           .sort
       end
     end

data/lib/rails_ai_context/tools/runtime_info.rb

@@ -203,7 +203,7 @@ module RailsAiContext
         end
 
         def gather_pending_migrations
-          RailsAiContext::PendingMigrations.live(File.join(rails_app.root, "db/migrate"))
+          RailsAiContext::PendingMigrations.live(RailsAiContext::PendingMigrations.migrate_dirs_for(rails_app.root))
         end
 
         def gather_index_usage(conn, adapter)

data/lib/rails_ai_context/tools/safe_call.rb

@@ -39,7 +39,8 @@ module RailsAiContext
         # nothing is reported.
         RailsAiContext::CodeReloader.with_app_code do
           begin
-            append_note(super(**kwargs), invalid_detail_note(discarded))
+            # One call is one run: what it reads per controller or per route is read once.
+            append_note(RailsAiContext::RunCache.around { super(**kwargs) }, invalid_detail_note(discarded))
           rescue StandardError => e
             # A failed call must not leak its recorded params into the next
             # call's session entry.
@@ -98,29 +99,33 @@ module RailsAiContext
 
       # A backtrace frame is an absolute path, and on an installed gem that is
       # the machine's GEM_HOME. The answer leaves the machine, so the frame is
-      # spelled the way every other path this gem reports is.
+      # spelled the way every other path this gem reports is, and one no
+      # portable form names keeps only its file name.
       def portable_origin(frame)
+        # Ruby 3.3+ names an eval without a file `(eval at FILE:LINE):LINE`.
+        if (eval_frame = frame.match(/\A\(eval at (.+)\)(:\d+(?::.*)?)?\z/m))
+          return "(eval at #{portable_origin(eval_frame[1])})#{eval_frame[2]}"
+        end
+
         path, rest = frame.split(":", 2)
-        return frame if path.to_s.empty?
+        # A relative path (`(eval)`, `-e`) names nothing on this machine.
+        return frame unless File.absolute_path?(path.to_s)
 
-        # No app, or one with no root: relativize still strips the gem prefix,
-        # so an empty root is an answer.
+        # This runs inside the rescue that answers a tool failure, so a raise
+        # here would leave the net and reach the client as a protocol error.
+        # No app, or no root, still strips the gem prefix.
         root = begin
           rails_app.root.to_s
-        rescue NameError
+        rescue StandardError
           ""
         end
-        [ RailsAiContext::PortablePath.relativize(path, root), rest ].compact.join(":")
-      rescue StandardError
-        # This runs inside the rescue that answers a tool failure, so a raise
-        # here leaves the net entirely and the client gets a protocol error.
-        # An empty root is the same answer the no-app branch above gives, and
-        # it still strips the prefix that names the machine.
-        begin
-          [ RailsAiContext::PortablePath.relativize(path, ""), rest ].compact.join(":")
+        relative = begin
+          RailsAiContext::PortablePath.relativize(path, root)
         rescue StandardError
-          frame
+          path
         end
+        relative = File.basename(relative) if File.absolute_path?(relative)
+        [ relative, rest ].compact.join(":")
       end
 
       # The value the caller sent, when this tool takes a DetailLevel detail