railwatch 0.1.0

data/lib/railwatch/subscribers/logs.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+module Railwatch
+  module Subscribers
+    # Captures Rails.logger lines (via a Logger subclass swap on the broadcast
+    # logger) and Rails 8.1 structured events (Rails.event). Level filter
+    # comes from config.log_level.
+    module Logs
+      extend Base
+
+      LEVELS = %w[debug info warn error fatal unknown].freeze
+      # Computed once so the hot log record doesn't look this up per call.
+      LOG_VERSION = Record::VERSIONS.fetch(:log)
+      # Lines Rails itself logs per request/job; the request and job records
+      # already carry this information, so they are not stored as logs.
+      FRAMEWORK_NOISE = /\A\s*(Started [A-Z]+ "|Processing by |Completed \d{3} |Parameters: \{|Rendered |Rendering |Performing |Performed |Enqueued |\[ActiveJob\]|Cannot render console)/.freeze
+      # Literal prefixes for the lines FRAMEWORK_NOISE matches, checked first
+      # with cheap start_with? so the regex only runs on lines that could
+      # plausibly match, instead of on every captured log line.
+      FRAMEWORK_NOISE_PREFIXES = %w[Started Processing Completed Parameters Rendered Rendering Performing Performed Enqueued [ActiveJob] Cannot].freeze
+
+      # Rails 8.1's own structured events (one per request/job phase); the
+      # request and job records already carry this information, so they are
+      # dropped unless an app opts in via config.capture_framework_events.
+      FRAMEWORK_EVENT_PREFIXES = %w[action_controller. action_dispatch. active_job. active_record.
+                                    action_view. action_mailer. active_storage. action_cable.].freeze
+
+      class Capture < ::Logger
+        # BroadcastLogger#debug? is true when ANY broadcast is at DEBUG, and
+        # every framework LogSubscriber (Active Record's SQL line, Action
+        # View's render lines, the cache store's) formats its message only
+        # when it is. A Logger.new(nil) sits at DEBUG, which made adding
+        # Railwatch turn all of that formatting on for lines nobody stored.
+        # Starting at config.log_level instead keeps the app's own level in
+        # charge. From here on this is an ordinary Logger: `Rails.logger.level
+        # =` and `Rails.logger.silence` reach it like any other broadcast, so
+        # what the app reads back is what it set.
+        def initialize
+          super(nil, level: Logs.min_severity)
+        end
+
+        def add(severity, message = nil, progname = nil)
+          return true unless Railwatch.enabled?
+          return true unless Logs.execution
+          severity ||= ::Logger::UNKNOWN
+          # Both gates: the Logger's own level (which the app may have raised
+          # or silenced) and config.log_level (which may have changed since
+          # this logger was built).
+          return true if severity < level || severity < Logs.min_severity
+          message = progname if message.nil? && !block_given?
+          message = yield if message.nil? && block_given?
+          Logs.write(LEVELS[severity] || "unknown", message)
+          true
+        end
+      end
+
+      class EventSubscriber
+        def emit(event)
+          return unless Railwatch.enabled?
+          name = event[:name].to_s
+          return if !Railwatch.config.capture_framework_events && FRAMEWORK_EVENT_PREFIXES.any? { |p| name.start_with?(p) }
+          Railwatch.record(:log,
+            group: Record.group_hash(name),
+            timestamp: event[:timestamp] ? event[:timestamp] / 1_000_000_000.0 : nil,
+            level: "event",
+            message: name,
+            tags: Array(event[:tags]).map(&:to_s),
+            context: (JSON.generate(event[:payload]) rescue "{}")[0, 8192],
+            source: event.dig(:source_location, :filepath) && "#{event[:source_location][:filepath].delete_prefix(Backtrace.app_root)}:#{event[:source_location][:lineno]}")
+        end
+      end
+
+      # Keyed by config.log_level so a level change is picked up on the next
+      # log line, but the common case (level unchanged) is one hash lookup
+      # instead of a #to_s allocation plus a LEVELS#index scan per line.
+      @min_severity_cache = {}
+
+      module_function
+
+      def install!(_app)
+        logger = Rails.logger
+        if logger.respond_to?(:broadcast_to)
+          logger.broadcast_to(Capture.new)
+        elsif logger.respond_to?(:extend)
+          logger.extend(ActiveSupport::BroadcastLogger) rescue nil
+          logger.broadcast_to(Capture.new) if logger.respond_to?(:broadcast_to)
+        end
+        Rails.event.subscribe(EventSubscriber.new) if Rails.respond_to?(:event)
+      end
+
+      def min_severity
+        level = Railwatch.config.log_level
+        @min_severity_cache[level] ||= (LEVELS.index(level.to_s) || 1)
+      end
+
+      # Cheap literal check before the FRAMEWORK_NOISE regex: most log lines
+      # share no prefix with the noise patterns, so this skips the regex
+      # engine entirely for them.
+      def framework_noise?(text)
+        text = text.lstrip if text.start_with?(" ", "\t")
+        FRAMEWORK_NOISE_PREFIXES.any? { |p| text.start_with?(p) } && text.match?(FRAMEWORK_NOISE)
+      end
+
+      def write(level, message)
+        exe = execution or return
+        text = message.is_a?(String) ? message : message.inspect
+        text = text.gsub(/\e\[[\d;]*m/, "") if text.include?("\e")
+        return if text.start_with?("[railwatch]") || framework_noise?(text)
+        exe.count(:logs)
+        return unless recording?
+        cfg = Railwatch.config
+        # One hash literal instead of kwargs-packing into Railwatch.record --
+        # every captured Rails.logger line goes through here.
+        Railwatch.push(:log, {
+          v: LOG_VERSION,
+          t: "log",
+          timestamp: Clock.now,
+          deploy: cfg.deploy,
+          server: cfg.server,
+          _group: nil,
+          **exe.envelope,
+          level: level,
+          message: text[0, 8192],
+          tags: current_tags,
+          context: Context.serialized
+        })
+      end
+
+      def current_tags
+        logger = Rails.logger
+        logger.respond_to?(:current_tags) ? logger.current_tags.map(&:to_s) : []
+      rescue StandardError
+        []
+      end
+    end
+  end
+end

data/lib/railwatch/subscribers/mail.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Railwatch
+  module Subscribers
+    # process.action_mailer (render) and deliver.action_mailer (send).
+    module Mail
+      extend Base
+
+      module_function
+
+      def install!(_app)
+        subscribe("deliver.action_mailer") do |event|
+          exe = execution
+          exe&.count(:mail)
+          next unless recording?
+          p = event.payload
+          mailer = p[:mailer].to_s
+          Railwatch.record(:mail,
+            group: Record.group_hash(mailer),
+            timestamp: started_at(event),
+            mailer: mailer,
+            subject: p[:subject].to_s[0, 255],
+            to: Array(p[:to]).size, cc: Array(p[:cc]).size, bcc: Array(p[:bcc]).size,
+            attachments: (p[:mail]&.attachments&.size rescue 0),
+            delivery_method: (p[:mail]&.delivery_method&.class&.name&.demodulize rescue nil),
+            perform_deliveries: p[:perform_deliveries] != false,
+            duration: micros(event),
+            failed: p[:exception].present?,
+            message_id: p[:message_id].to_s[0, 255])
+        end
+
+        subscribe("process.action_mailer") do |event|
+          next unless recording?
+          p = event.payload
+          Railwatch.record(:view_render, group: Record.group_hash("mailer", p[:mailer], p[:action]),
+                         timestamp: started_at(event), identifier: "#{p[:mailer]}##{p[:action]}",
+                         kind: "mailer", duration: micros(event))
+        end
+      end
+    end
+  end
+end

data/lib/railwatch/subscribers/notifications.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Railwatch
+  module Subscribers
+    # Noticed gem deliveries, only when the gem is loaded. Noticed delivery
+    # methods are Active Jobs, so we hook their perform via the Jobs
+    # subscriber and tag them here.
+    module Notifications
+      extend Base
+
+      module_function
+
+      def install!(_app)
+        return unless defined?(::Noticed)
+
+        ActiveSupport::Notifications.subscribe("perform.active_job") do |event|
+          job = event.payload[:job]
+          next unless job.class.name.to_s.start_with?("Noticed::")
+          exe = execution
+          exe&.count(:notifications)
+          next unless recording?
+          Railwatch.record(:notification,
+            group: Record.group_hash(job.class.name),
+            timestamp: started_at(event),
+            notifier: (job.arguments.first.is_a?(Hash) ? job.arguments.first[:notification_class] : nil).to_s,
+            delivery_method: job.class.name.demodulize,
+            channel: job.class.name.demodulize.delete_suffix("Delivery").downcase,
+            duration: micros(event),
+            failed: event.payload[:exception].present?)
+        rescue StandardError => e
+          Railwatch.debug { "noticed subscriber: #{e.message}" }
+        end
+      end
+    end
+  end
+end

data/lib/railwatch/subscribers/process_info.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module Railwatch
+  module Subscribers
+    # One `process` record per process boot: role (web / worker / console /
+    # command), Ruby and Rails versions, boot time. Gives the platform a
+    # server and deploy inventory for free.
+    module ProcessInfo
+      extend Base
+
+      module_function
+
+      # Keeps the app; the record itself is written from the engine's
+      # after_initialize, once the app is fully booted, so boot_seconds
+      # covers the app's own initializers rather than stopping short of
+      # them.
+      def install!(app)
+        @app = app
+        @database_adapter = nil
+      end
+
+      def restart_after_fork!
+        record!
+      end
+
+      def record!
+        boot = Clock.monotonic - Railwatch::BOOTED_AT
+        Railwatch.record(:process,
+          pid: Process.pid,
+          role: role,
+          ruby_version: RUBY_VERSION,
+          rails_version: (Rails.version rescue nil),
+          railwatch_version: Railwatch::VERSION,
+          app: @app&.class&.module_parent_name,
+          environment: Railwatch.config.environment_name,
+          boot_seconds: boot,
+          database_adapter: database_adapter,
+          queue_adapter: queue_adapter,
+          cache_store: (Rails.cache.class.name rescue nil))
+      end
+
+      # Resolved from the app's configuration with Rails' own resolvers
+      # rather than through ActiveRecord::Base / ActiveJob::Base: in a
+      # lazy-loading process (development, or a production boot without
+      # eager loading) touching those constants is what loads the
+      # frameworks, some 350 ms of boot nothing else asked for. Both
+      # resolvers are requirable on their own.
+      # Memoised: database.yml is re-read and re-rendered on every
+      # database_configuration call, and the adapter cannot change across a
+      # fork. Keyed by Rails.env, as Active Record itself does; DATABASE_URL
+      # and `url:` entries are resolved the same way it resolves them.
+      def database_adapter
+        @database_adapter ||= begin
+          config = @app&.config&.database_configuration
+          if config
+            require "active_record/database_configurations"
+            ActiveRecord::DatabaseConfigurations.new(config).find_db_config(Rails.env)&.adapter
+          end
+        rescue StandardError
+          nil
+        end
+      end
+
+      # The effective adapter lives on ActiveJob::Base, which an app may set
+      # directly in an initializer; it is read from there whenever the
+      # framework is already loaded (any eager-loaded app with jobs), and
+      # from configuration otherwise, so a lazy boot does not load Active
+      # Job for one string.
+      def queue_adapter
+        return ActiveJob::Base.queue_adapter_name if ActiveJob.autoload?(:Base).nil?
+
+        configured_queue_adapter
+      rescue StandardError
+        nil
+      end
+
+      def configured_queue_adapter
+        adapter = @app&.config&.active_job&.queue_adapter or return nil
+        return adapter.to_s if adapter.is_a?(Symbol) || adapter.is_a?(String)
+
+        require "active_job/queue_adapter"
+        ActiveJob.adapter_name(adapter).underscore
+      end
+
+      # Puma is loaded in every process of an app that bundles it, so a Solid
+      # Queue worker is recognised first, by how it was started (bin/jobs or
+      # `rake solid_queue:start`).
+      def role
+        if defined?(::SolidQueue) && ($PROGRAM_NAME.include?("jobs") || ARGV.first.to_s.start_with?("solid_queue:")) then "worker"
+        elsif defined?(::Rails::Console) then "console"
+        elsif $PROGRAM_NAME.end_with?("rake") then "command"
+        elsif defined?(::Puma) then "web"
+        else "process"
+        end
+      end
+    end
+  end
+end

data/lib/railwatch/subscribers/queries.rb

@@ -0,0 +1,183 @@
+# frozen_string_literal: true
+
+module Railwatch
+  module Subscribers
+    # sql.active_record, transactions, model hydration, strict loading.
+    # Source location is only computed for slow queries or the first time a
+    # group is seen in an execution, to keep the per-query cost tiny.
+    module Queries
+      extend Base
+
+      SKIP_NAMES = %w[SCHEMA TRANSACTION EXPLAIN].freeze
+      MAX_SQL = 16_384
+      # Computed once so the hot query record doesn't look these up per call.
+      QUERY_VERSION = Record::VERSIONS.fetch(:query)
+
+      MAX_EXPLAIN = 4_000
+      EXPLAIN_TTL = 600
+      MAX_EXPLAINED_GROUPS = 1_000
+      SELECT = /\A\s*select\b/i
+
+      @connection_info = {}.compare_by_identity
+      @sources = {}
+      @explained = {}
+
+      module_function
+
+      # adapter name and db config name never change for a connection object.
+      # Adapter, database name, and the multi-DB role the connection was
+      # checked out for ("writing"/"reading"; Nightwatch calls this the
+      # connection type). Memoised per connection object.
+      def connection_info(conn)
+        @connection_info[conn] ||= begin
+          adapter = conn.adapter_name.to_s.downcase
+          db = (conn.pool.db_config.name rescue nil)
+          role = (conn.role.to_s rescue "writing")
+          [ adapter, db, role ].freeze
+        end
+      rescue StandardError
+        [ "", nil, "writing" ].freeze
+      end
+
+      # The app frame that issues a query shape rarely changes, so the
+      # (expensive) caller walk runs once per group per process.
+      def source_for(group, slow)
+        return @sources[group] if @sources.key?(group) && !slow
+        loc = Backtrace.caller_location(skip: 4)
+        @sources.clear if @sources.size > 5_000
+        @sources[group] = loc
+        loc
+      end
+
+      # The query plan for a slow SELECT, off by default. Runs on the very
+      # connection that just ran the query -- the adapter's own #explain, so
+      # each database gets its native plan format -- with Railwatch paused, and
+      # with the EXPLAIN's own sql.active_record notification named "EXPLAIN"
+      # and therefore already dropped by SKIP_NAMES above. Re-entering the
+      # connection here is safe: the notification fires after the outer
+      # statement's result has been materialized, and the adapter lock is
+      # reentrant.
+      def explain_for(payload, duration_ms, group)
+        return nil unless duration_ms >= Railwatch.config.explain_threshold_ms
+        conn = payload[:connection] or return nil
+        return nil unless SELECT.match?(payload[:sql])
+        return nil unless explain_due?(group)
+
+        plan = Railwatch.ignore { conn.explain(payload[:sql], payload[:binds] || []) }
+        plan&.to_s&.slice(0, MAX_EXPLAIN)
+      rescue StandardError
+        nil
+      end
+
+      # One EXPLAIN per query shape per process per EXPLAIN_TTL. Racy across
+      # threads by design (worst case two threads explain the same shape
+      # once), like @sources above.
+      def explain_due?(group)
+        now = Clock.monotonic
+        last = @explained[group]
+        return false if last && now - last < EXPLAIN_TTL
+        @explained.clear if @explained.size >= MAX_EXPLAINED_GROUPS
+        @explained[group] = now
+        true
+      end
+
+      def install!(_app)
+        subscribe("sql.active_record") do |event|
+          p = event.payload
+          next if SKIP_NAMES.include?(p[:name])
+          exe = execution
+          if p[:cached]
+            exe&.count(:cached_queries)
+            next
+          end
+          exe&.count(:queries)
+          # Statement count for the currently-open transaction, keyed by AR's
+          # transaction object identity (the same object the transaction
+          # subscriber below sees), so nested/concurrent transactions on the
+          # same execution don't collide. Read once at transaction end.
+          exe&.count_transaction_statement(p[:transaction].object_id) if p[:transaction]
+          next unless recording?
+
+          sql = p[:sql]
+          adapter, db, role = p[:connection] ? connection_info(p[:connection]) : [ "", nil, "writing" ]
+          # One cache lookup gets both the group hash and the normalized SQL,
+          # so the (rare) n+1 branch below never re-normalizes the same text.
+          group, normalized = SqlNormalizer.group_and_normalized(sql, adapter: adapter, connection_name: db)
+          exe&.track_query_group(group)
+          duration = micros(event)
+          cfg = Railwatch.config
+          slow = event.duration >= cfg.slow_query_threshold_ms
+          n = exe ? exe.query_groups[group] : 0
+
+          # One hash literal (base keys + envelope + fields) instead of
+          # kwargs-packing into Railwatch.record and merging through
+          # Record.build -- this is the hottest record type in the gem.
+          Railwatch.push(:query, {
+            v: QUERY_VERSION,
+            t: "query",
+            timestamp: started_at(event),
+            deploy: cfg.deploy,
+            server: cfg.server,
+            _group: group,
+            **(exe ? exe.envelope : Record::EMPTY_ENVELOPE),
+            sql: cfg.capture_sql_values ? SqlNormalizer.raw_for_record(sql, max_characters: MAX_SQL) : normalized[0, MAX_SQL],
+            name: p[:name],
+            duration: duration,
+            connection: db,
+            adapter: adapter,
+            role: role,
+            async: p[:async] ? true : false,
+            row_count: p[:row_count],
+            affected_rows: p[:affected_rows],
+            in_transaction: p[:transaction] ? true : false,
+            source: source_for(group, slow),
+            allocations: event.allocations,
+            # The EXPLAIN still runs on the raw statement -- the plan would be
+            # meaningless otherwise -- and capture_query_explain keeps working
+            # on its own. Only what is STORED changes: `sql` above is the
+            # normalized shape. A plan can echo literal predicates (Postgres
+            # does, in Filter/Index Cond lines), so capture_query_explain is
+            # its own privacy decision, independent of capture_sql_values;
+            # docs/configuration.md says so at both options.
+            explain: cfg.capture_query_explain ? explain_for(p, event.duration, group) : nil
+          })
+
+          if exe && n == cfg.n_plus_one_threshold
+            Railwatch.record(:n_plus_one, group: group, sql: normalized[0, 2048],
+                           count: n, source: Backtrace.caller_location(skip: 3))
+          end
+        end
+
+        subscribe("transaction.active_record") do |event|
+          exe = execution
+          exe&.count(:transactions)
+          p = event.payload
+          # Taken (and removed) whether or not a record is built: the
+          # statement counter above runs for every execution, so this is what
+          # keeps a sampled-out execution from holding one entry per
+          # transaction for its whole lifetime.
+          statement_count = p[:transaction] ? exe&.transaction_statement_count(p[:transaction].object_id) : nil
+          next unless recording?
+          connection_name = (p[:connection]&.pool&.db_config&.name rescue nil)
+          outcome = p[:outcome].to_s
+          Railwatch.record(:transaction, group: Record.group_hash(connection_name, outcome),
+                         timestamp: started_at(event), duration: micros(event), outcome: outcome,
+                         connection: connection_name, statement_count: statement_count)
+        end
+
+        subscribe_payload("instantiation.active_record") do |payload|
+          execution&.count(:hydrated_models, payload[:record_count].to_i)
+        end
+
+        subscribe("strict_loading_violation.active_record") do |event|
+          exe = execution or next
+          exe.count(:lazy_loads)
+          next unless recording?
+          p = event.payload
+          Railwatch.record(:log, level: "warn", message: "Lazy load of #{p[:owner].class.name}##{p[:reflection]&.name}",
+                         tags: [ "strict_loading" ], context: "{}", source: Backtrace.caller_location(skip: 3))
+        end
+      end
+    end
+  end
+end

data/lib/railwatch/subscribers/requests.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module Railwatch
+  module Subscribers
+    # Controller lifecycle. The Rack middleware owns the request record; this
+    # subscriber fills in route, stage boundaries, and the extras Rails
+    # exposes (redirects, halted callbacks, unpermitted params, rate limits).
+    module Requests
+      extend Base
+
+      # Only a multipart request can carry an UploadedFile, so the (recursive)
+      # params walk below is skipped entirely for the vast majority of
+      # requests instead of running once per action.
+      EMPTY_FILES = [].freeze
+
+      module_function
+
+      def install!(_app)
+        subscribe("start_processing.action_controller") do |event|
+          exe = execution or next
+          p = event.payload
+          exe.enter_stage(:action)
+          env = p[:request]&.env
+          next unless env
+          env["railwatch.route"] = { pattern: route_pattern(p[:request]), controller: p[:controller].to_s.delete_suffix("Controller").underscore, action: p[:action], verb: p[:method] }
+          exe.preview = "#{p[:controller]}##{p[:action]}"
+          # Captured here, not in the closing middleware, because Rack's
+          # TempfileReaper (nested inside us) has already closed and unlinked
+          # upload tempfiles by the time the middleware unwinds.
+          env["railwatch.files"] = env["CONTENT_TYPE"]&.start_with?("multipart/form-data") ? uploaded_files(p[:request].params) : EMPTY_FILES
+        end
+
+        subscribe("process_action.action_controller") do |event|
+          exe = execution or next
+          p = event.payload
+          env = p[:request]&.env or next
+          env["railwatch.view_runtime"] = p[:view_runtime]&.round(2)
+          env["railwatch.db_runtime"] = p[:db_runtime]&.round(2)
+          exe.enter_stage(:middleware_after)
+          resp = p[:response]
+          if resp && env["HTTP_X_INERTIA"] == "true"
+            env["railwatch.inertia_props_bytes"] = resp.body.bytesize rescue nil
+          end
+        end
+
+        subscribe("redirect_to.action_controller") do |event|
+          env = event.payload[:request]&.env or next
+          env["railwatch.redirect_to"] = Record.url_without_sensitive_components(event.payload[:location], limit: 512)
+        end
+
+        subscribe("halted_callback.action_controller") do |event|
+          exe = execution or next
+          ActiveSupport::ExecutionContext.to_h[:controller]&.request&.env&.[]=("railwatch.halted_callback", event.payload[:filter].to_s)
+        end
+
+        subscribe("unpermitted_parameters.action_controller") do |event|
+          env = event.payload.dig(:context, :request)&.env or next
+          env["railwatch.unpermitted_parameters"] = Array(event.payload[:keys]).map(&:to_s)
+        end
+
+        subscribe("rate_limit.action_controller") do |event|
+          env = event.payload[:request]&.env or next
+          env["railwatch.rate_limited"] = { name: event.payload[:name].to_s, count: event.payload[:count], to: event.payload[:to] }
+        end
+
+        subscribe("render_template.action_view") do |_event|
+          exe = execution or next
+          exe.enter_stage(:render) if exe.stage == :action
+        end
+      end
+
+      def route_pattern(request)
+        request.route_uri_pattern
+      rescue StandardError
+        nil
+      end
+
+      def uploaded_files(value, name = nil)
+        case value
+        when ActionDispatch::Http::UploadedFile
+          [ { name: name, size: (value.tempfile.size rescue nil), content_type: value.content_type, error: nil } ]
+        when Hash
+          value.flat_map { |k, v| uploaded_files(v, k.to_s) }
+        when Array
+          value.flat_map { |v| uploaded_files(v, name) }
+        else
+          []
+        end
+      rescue StandardError
+        []
+      end
+    end
+  end
+end

data/lib/railwatch/subscribers/storage.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Railwatch
+  module Subscribers
+    module Storage
+      extend Base
+
+      OPS = %w[service_upload service_download service_streaming_download service_delete
+               service_delete_prefixed service_exist service_url service_update_metadata
+               analyze transform preview].freeze
+
+      module_function
+
+      def install!(_app)
+        OPS.each do |op|
+          subscribe("#{op}.active_storage") do |event|
+            exe = execution
+            exe&.count(:storage_ops)
+            next unless recording?
+            p = event.payload
+            service = p[:service].to_s
+            Railwatch.record(:storage_op,
+              group: Record.group_hash(service, op),
+              timestamp: started_at(event),
+              service: service,
+              op: op.delete_prefix("service_"),
+              key: p[:key].to_s[0, 255],
+              duration: micros(event),
+              exist: p[:exist])
+          end
+        end
+      end
+    end
+  end
+end